From 72e4012489228ddea48774f0ea4dd35a06505699 Mon Sep 17 00:00:00 2001 From: Alexander Huddleston Date: Sat, 25 Apr 2020 15:52:51 -0500 Subject: [PATCH] Archiving to Nextcloud successfully, added some error checking and handling. --- Pipfile | 3 ++- lib/TweetStreamer.py | 11 +++++++++- lib/archival.py | 12 +++++++---- lib/authentication.py | 26 ++++++++++++++++++----- lib/echo_nextcloud.py | 48 +++++++++++++++++++++++++++++++++++++++++++ template_config.json | 26 ++++++++++++++++++----- twitter_media_tool.py | 14 +++++++++---- 7 files changed, 120 insertions(+), 20 deletions(-) create mode 100644 lib/echo_nextcloud.py diff --git a/Pipfile b/Pipfile index 4f97087..930c4a3 100644 --- a/Pipfile +++ b/Pipfile @@ -10,6 +10,7 @@ pipfile = "*" [packages] pipfile = "*" -"mastodon.py" = "*" tweepy = "*" python-twitter = "*" +"mastodon.py" = "*" +webdavclient3 = "*" diff --git a/lib/TweetStreamer.py b/lib/TweetStreamer.py index 99712a0..e021ef7 100644 --- a/lib/TweetStreamer.py +++ b/lib/TweetStreamer.py @@ -2,10 +2,19 @@ from tweepy import StreamListener from lib.archival import archive_media_status +from lib.echo_nextcloud import nextcloud_upload_media class TweetStreamer(StreamListener): def on_status(self, status): print(status._json) - archive_media_status(status) + archive_filenames = archive_media_status(status) + if(archive_filenames): + for filename in archive_filenames: + print(filename) + try: + nextcloud_upload_media(filename, status.timestamp_ms) + except Exception as e: + print('Was unsuccessful in uploading the file.') + print(e) return super().on_status(status) diff --git a/lib/archival.py b/lib/archival.py index f6c640a..73d7449 100644 --- a/lib/archival.py +++ b/lib/archival.py @@ -1,7 +1,6 @@ #!/usr/bin/python3 from twitter.twitter_utils import parse_media_file -# from twitter import Api def archive_media(media_url): @@ -11,6 +10,7 @@ def archive_media(media_url): archive_file.writelines(temp_media_file.readlines()) archive_file.close() temp_media_file.close() + return filename def archive_media_status(status): @@ -18,11 +18,15 @@ def archive_media_status(status): if(not ('retweeted_status' in status._json.keys())): if('extended_entities' in status._json.keys()): if('media' in status.extended_entities.keys()): + archive_filenames = [] for media_dict in status.extended_entities['media']: if(media_dict['type'] == 'photo'): - archive_media(media_dict['media_url']) + archive_filenames.append( + archive_media(media_dict['media_url'])) elif(media_dict['type'] == 'video'): - archive_media( - media_dict['video_info']['variants'][-1:][0]['url']) + archive_filenames.append(archive_media( + media_dict['video_info']['variants'][-1:][0]['url'])) + return archive_filenames except Exception as e: print(e) + return '' diff --git a/lib/authentication.py b/lib/authentication.py index 6650b10..0e84940 100644 --- a/lib/authentication.py +++ b/lib/authentication.py @@ -1,11 +1,27 @@ #!/usr/bin/python3 from tweepy import OAuthHandler, API +from webdav3.client import Client def authenticate_twitter(config): - auth = OAuthHandler(config['api_key'], config['api_key_secret']) - auth.set_access_token(config['access_token'], - config['access_token_secret']) - twitter_api = API(auth) - return twitter_api + try: + auth = OAuthHandler(config['api_key'], config['api_key_secret']) + auth.set_access_token(config['access_token'], + config['access_token_secret']) + twitter_api = API(auth) + return twitter_api + except Exception as e: + print('There was some error attempting to authenticate with Twitter API') + print(f'\t{e}') + exit(1) + + +def authenticate_nextcloud(config): + try: + client = Client(config) + return client + except Exception as e: + print('There was some error attempting to authenticate through WebDAV') + print(f'\t{e}') + exit(1) diff --git a/lib/echo_nextcloud.py b/lib/echo_nextcloud.py new file mode 100644 index 0000000..d2c166c --- /dev/null +++ b/lib/echo_nextcloud.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +from datetime import datetime as dt +from lib.setup import import_config_file +from lib.authentication import authenticate_nextcloud + + +def setup_archive_data(timestamp): + config = import_config_file() + if(config['nextcloud']['webdav_hostname'] and config['nextcloud']['webdav_login'] and config['nextcloud']['webdav_password']): + client = authenticate_nextcloud(config['nextcloud']) + else: + print( + 'Please fill out the necessary information for using Nextcloud through WebDAV.') + print('\thttps://docs.nextcloud.com/server/16/user_manual/files/access_webdav.html') + exit(1) + ts = dt.fromtimestamp(int(int(timestamp)/1000)) + date_ts = ts.strftime('%Y_%m_%d') + time_ts = ts.strftime('%H_%M_%S') + return client, config['nextcloud_upload_path'], date_ts, time_ts + + +def setup_archive_dir(client, upload_path, date_ts): + if(not client.check(upload_path)): + client.mkdir(f'{upload_path}') + client.mkdir(f'{upload_path}/{date_ts}') + else: + if(not client.check(f'{upload_path}/{date_ts}')): + client.mkdir(f'{upload_path}/{date_ts}') + + +def attempt_upload_media(client, upload_dir, upload_filename, upload_filetype, archive_filename): + file_dne = True + count = 1 + while(file_dne): + temp_filename = f'{upload_dir}/{upload_filename}_{count}.{upload_filetype}' + if(not client.check(temp_filename)): + client.upload_file(temp_filename, f'{archive_filename}') + file_dne = False + else: + count += 1 + + +def nextcloud_upload_media(archive_filename, timestamp): + client, upload_path, date_ts, time_ts = setup_archive_data(timestamp) + setup_archive_dir(client, upload_path, date_ts) + attempt_upload_media(client, f'{upload_path}/{date_ts}', + f'{date_ts}_{time_ts}', f'{archive_filename.split(".")[1]}', f'data/{archive_filename}') diff --git a/template_config.json b/template_config.json index 7ea0cb4..f6189c2 100644 --- a/template_config.json +++ b/template_config.json @@ -1,6 +1,22 @@ { - "api_key":"", - "api_key_secret":"", - "access_token":"", - "access_token_secret":"" -} \ No newline at end of file + "twitter": { + "api_key":"", + "api_key_secret":"", + "access_token":"", + "access_token_secret":"" + }, + "nextcloud": { + "webdav_hostname": "", + "webdav_login": "", + "webdav_password": "" + }, + "nextcloud_upload_path": "", + "mastodon": { + "api_key": "", + "api_key_secret": "" + }, + "discord": { + "api_key": "", + "api_key_secret": "" + } +} diff --git a/twitter_media_tool.py b/twitter_media_tool.py index 7e6c582..6235817 100644 --- a/twitter_media_tool.py +++ b/twitter_media_tool.py @@ -8,11 +8,17 @@ from tweepy import Stream def main(): config = import_config_file() - twitter_api = authenticate_twitter(config) - tweet_stream_listener = TweetStreamer(twitter_api) - tweet_stream = Stream(auth=twitter_api.auth, + if(config['twitter']['api_key'] and config['twitter']['api_key_secret'] and config['twitter']['access_token'] and config['twitter']['access_token_secret']): + twitter_client = authenticate_twitter(config['twitter']) + else: + print('Please fill out the necessary information for using the Twitter API.') + print('\thttps://developer.twitter.com') + exit(1) + tweet_stream_listener = TweetStreamer(twitter_client) + tweet_stream = Stream(auth=twitter_client.auth, listener=tweet_stream_listener) - tweet_stream.filter(follow=[twitter_api.me()._json['id_str']]) + tweet_stream.filter( + follow=[twitter_client.me()._json['id_str']], is_async=True) if(__name__ == '__main__'):