Added proper docstrings to all created methods for better self-documentation.
This commit is contained in:
parent
154bf4c3e4
commit
c437febbed
6 changed files with 137 additions and 2 deletions
|
@ -9,7 +9,23 @@ import logging
|
||||||
|
|
||||||
|
|
||||||
class TweetStreamer(StreamListener):
|
class TweetStreamer(StreamListener):
|
||||||
|
"""
|
||||||
|
TweetStreamer class definition
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
StreamListener {obj} -- Tweepy API StreamListener object
|
||||||
|
"""
|
||||||
|
|
||||||
def on_status(self, status):
|
def on_status(self, status):
|
||||||
|
"""
|
||||||
|
Method definition for what happens when a new status is made
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
status {obj} -- Tweepy "Status" object
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
super -- on_status() super method
|
||||||
|
"""
|
||||||
print(status._json)
|
print(status._json)
|
||||||
archive_filenames = archive_media_status(status)
|
archive_filenames = archive_media_status(status)
|
||||||
if(archive_filenames):
|
if(archive_filenames):
|
||||||
|
|
|
@ -4,6 +4,16 @@ from twitter.twitter_utils import parse_media_file
|
||||||
|
|
||||||
|
|
||||||
def archive_media(media_url):
|
def archive_media(media_url):
|
||||||
|
"""
|
||||||
|
Parse the media file from the URL using Twitter API's parse_media_file method,
|
||||||
|
then write the resulting temporary file to data/[filename].
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
media_url {str} -- String noting the URL of the uploaded media file
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str -- Resulting filename
|
||||||
|
"""
|
||||||
temp_media_file, filename, size, media_type = parse_media_file(
|
temp_media_file, filename, size, media_type = parse_media_file(
|
||||||
media_url)
|
media_url)
|
||||||
with open(f'data/{filename}', 'bw+') as archive_file:
|
with open(f'data/{filename}', 'bw+') as archive_file:
|
||||||
|
@ -14,6 +24,15 @@ def archive_media(media_url):
|
||||||
|
|
||||||
|
|
||||||
def filter_out_bad_content_type(video_variant_list):
|
def filter_out_bad_content_type(video_variant_list):
|
||||||
|
"""
|
||||||
|
Helper method to filter out unusable video variants from the list
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
video_variant_list {list} -- List of video variant definitions from the tweet
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
list -- Resulting filtered video variant list
|
||||||
|
"""
|
||||||
filtered_variant_list = []
|
filtered_variant_list = []
|
||||||
for video in video_variant_list:
|
for video in video_variant_list:
|
||||||
if(video['content_type'] == 'video/mp4'):
|
if(video['content_type'] == 'video/mp4'):
|
||||||
|
@ -22,6 +41,15 @@ def filter_out_bad_content_type(video_variant_list):
|
||||||
|
|
||||||
|
|
||||||
def select_video_variant(video_variant_list):
|
def select_video_variant(video_variant_list):
|
||||||
|
"""
|
||||||
|
Iterate the video variants and pick the highest quality variant
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
video_variant_list {list} -- List of video variant definitions from the tweet
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str -- Resulting URL for the highest quality video variant
|
||||||
|
"""
|
||||||
filtered_variant_list = filter_out_bad_content_type(video_variant_list)
|
filtered_variant_list = filter_out_bad_content_type(video_variant_list)
|
||||||
highest_bitrate_variant = filtered_variant_list[0]
|
highest_bitrate_variant = filtered_variant_list[0]
|
||||||
for video in filtered_variant_list:
|
for video in filtered_variant_list:
|
||||||
|
@ -31,6 +59,15 @@ def select_video_variant(video_variant_list):
|
||||||
|
|
||||||
|
|
||||||
def archive_media_status(status):
|
def archive_media_status(status):
|
||||||
|
"""
|
||||||
|
Locally archive any media from the given Tweepy API "Status" object
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
status {obj} -- Tweepy API "Status" object
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str -- None, or a list of filenames of the media archived locally
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
if(not ('retweeted_status' in status._json.keys())):
|
if(not ('retweeted_status' in status._json.keys())):
|
||||||
if('extended_entities' in status._json.keys()):
|
if('extended_entities' in status._json.keys()):
|
||||||
|
@ -41,8 +78,8 @@ def archive_media_status(status):
|
||||||
archive_filenames.append(
|
archive_filenames.append(
|
||||||
archive_media(media_dict['media_url']))
|
archive_media(media_dict['media_url']))
|
||||||
elif(media_dict['type'] == 'video'):
|
elif(media_dict['type'] == 'video'):
|
||||||
archive_filenames.append(archive_media(
|
archive_filenames.append(
|
||||||
select_video_variant(media_dict['video_info']['variants'])))
|
archive_media(select_video_variant(media_dict['video_info']['variants'])))
|
||||||
return archive_filenames
|
return archive_filenames
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(e)
|
print(e)
|
||||||
|
|
|
@ -6,6 +6,15 @@ from mastodon import Mastodon as masto
|
||||||
|
|
||||||
|
|
||||||
def authenticate_twitter(config):
|
def authenticate_twitter(config):
|
||||||
|
"""
|
||||||
|
Authenticate with the Tweepy API and return client object
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
config {dict} -- Configuration dictionary object
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
obj -- Tweepy API client object
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
auth = OAuthHandler(config['api_key'], config['api_key_secret'])
|
auth = OAuthHandler(config['api_key'], config['api_key_secret'])
|
||||||
auth.set_access_token(config['access_token'],
|
auth.set_access_token(config['access_token'],
|
||||||
|
@ -19,6 +28,15 @@ def authenticate_twitter(config):
|
||||||
|
|
||||||
|
|
||||||
def authenticate_nextcloud(config):
|
def authenticate_nextcloud(config):
|
||||||
|
"""
|
||||||
|
Authenticate Nextcloud with WebDAV and return client object
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
config {dict} -- Configuration dictionary object
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
obj -- WebDAV client object
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
client = Client(config)
|
client = Client(config)
|
||||||
return client
|
return client
|
||||||
|
@ -29,6 +47,15 @@ def authenticate_nextcloud(config):
|
||||||
|
|
||||||
|
|
||||||
def authenticate_mastodon(config):
|
def authenticate_mastodon(config):
|
||||||
|
"""
|
||||||
|
Authenticate Mastodon API and return client object
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
config {dict} -- Configuration dictionary object
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
obj -- Mastodon API client object
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
client = masto(access_token=config['access_token'],
|
client = masto(access_token=config['access_token'],
|
||||||
api_base_url=config['api_base_url'])
|
api_base_url=config['api_base_url'])
|
||||||
|
|
|
@ -6,6 +6,18 @@ from lib.authentication import authenticate_nextcloud
|
||||||
|
|
||||||
|
|
||||||
def setup_archive_data(config, timestamp):
|
def setup_archive_data(config, timestamp):
|
||||||
|
"""
|
||||||
|
Set up the data necessary for media archival to Nextcloud
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
config {dict} -- Configuration dictionary object
|
||||||
|
timestamp {str} -- String describing the timestamp of the tweet
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
obj -- WebDAV client object
|
||||||
|
str -- String formatted date timestamp
|
||||||
|
str -- String formatted time timestamp
|
||||||
|
"""
|
||||||
if(config['webdav_hostname'] and config['webdav_login'] and config['webdav_password']):
|
if(config['webdav_hostname'] and config['webdav_login'] and config['webdav_password']):
|
||||||
client = authenticate_nextcloud(config)
|
client = authenticate_nextcloud(config)
|
||||||
else:
|
else:
|
||||||
|
@ -18,6 +30,14 @@ def setup_archive_data(config, timestamp):
|
||||||
|
|
||||||
|
|
||||||
def setup_archive_dir(client, upload_path, date_ts):
|
def setup_archive_dir(client, upload_path, date_ts):
|
||||||
|
"""
|
||||||
|
Set up Nextcloud directory path for file upload
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
client {obj} -- WebDAV client object
|
||||||
|
upload_path {str} -- Intended path to set up directories for upload archival
|
||||||
|
date_ts {str} -- String formatted date timestamp
|
||||||
|
"""
|
||||||
if(not client.check(upload_path)):
|
if(not client.check(upload_path)):
|
||||||
client.mkdir(f'{upload_path}')
|
client.mkdir(f'{upload_path}')
|
||||||
client.mkdir(f'{upload_path}/{date_ts}')
|
client.mkdir(f'{upload_path}/{date_ts}')
|
||||||
|
@ -27,6 +47,16 @@ def setup_archive_dir(client, upload_path, date_ts):
|
||||||
|
|
||||||
|
|
||||||
def attempt_upload_media(client, upload_dir, upload_filename, upload_filetype, archive_filename):
|
def attempt_upload_media(client, upload_dir, upload_filename, upload_filetype, archive_filename):
|
||||||
|
"""
|
||||||
|
Attempt to upload archived media to Nextcloud with WebDAV client
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
client {obj} -- WebDAV client object
|
||||||
|
upload_dir {str} -- String formatted directory path for Nextcloud archival
|
||||||
|
upload_filename {str} -- String formatted filename for Nextcloud archival
|
||||||
|
upload_filetype {str} -- File extension of archived media
|
||||||
|
archive_filename {str} -- String formatted filename of the local archive
|
||||||
|
"""
|
||||||
file_dne = True
|
file_dne = True
|
||||||
count = 1
|
count = 1
|
||||||
while(file_dne):
|
while(file_dne):
|
||||||
|
@ -39,6 +69,15 @@ def attempt_upload_media(client, upload_dir, upload_filename, upload_filetype, a
|
||||||
|
|
||||||
|
|
||||||
def nextcloud_upload_media(config, upload_path, archive_filename, timestamp):
|
def nextcloud_upload_media(config, upload_path, archive_filename, timestamp):
|
||||||
|
"""
|
||||||
|
Upload archived media file to Nextcloud with WebDAV client
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
config {obj} -- Configuration dictionary object
|
||||||
|
upload_path {str} -- Nextcloud upload directory path
|
||||||
|
archive_filename {str} -- String formatted filename of the local archive
|
||||||
|
timestamp {str} -- String describing the timestamp of the tweet
|
||||||
|
"""
|
||||||
client, date_ts, time_ts = setup_archive_data(config, timestamp)
|
client, date_ts, time_ts = setup_archive_data(config, timestamp)
|
||||||
setup_archive_dir(client, upload_path, date_ts)
|
setup_archive_dir(client, upload_path, date_ts)
|
||||||
attempt_upload_media(client, f'{upload_path}/{date_ts}',
|
attempt_upload_media(client, f'{upload_path}/{date_ts}',
|
||||||
|
|
13
lib/setup.py
13
lib/setup.py
|
@ -5,6 +5,12 @@ from os import environ
|
||||||
|
|
||||||
|
|
||||||
def import_from_default_path():
|
def import_from_default_path():
|
||||||
|
"""
|
||||||
|
Import the configuration JSON from the default path (config/config.json)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict -- Configuration dictionary object
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
with open('config/config.json') as config_file:
|
with open('config/config.json') as config_file:
|
||||||
return load(config_file)
|
return load(config_file)
|
||||||
|
@ -15,6 +21,13 @@ def import_from_default_path():
|
||||||
|
|
||||||
|
|
||||||
def import_config_file():
|
def import_config_file():
|
||||||
|
"""
|
||||||
|
Import the configuration JSON from the path specified in TMT_CONFIG_PATH, otherwise default path
|
||||||
|
(i.e. call import_from_default_path())
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict -- Configuration dictionary object
|
||||||
|
"""
|
||||||
if('TMT_CONFIG_PATH' in environ.keys()):
|
if('TMT_CONFIG_PATH' in environ.keys()):
|
||||||
try:
|
try:
|
||||||
with open(f'{environ["TMT_CONFIG_PATH"]}') as config_file:
|
with open(f'{environ["TMT_CONFIG_PATH"]}') as config_file:
|
||||||
|
|
|
@ -7,6 +7,9 @@ from tweepy import Stream
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
"""
|
||||||
|
Initialize and run the TweetStreamer and listen for new tweets
|
||||||
|
"""
|
||||||
config = import_config_file()
|
config = import_config_file()
|
||||||
if(config['twitter']['api_key'] and config['twitter']['api_key_secret'] and config['twitter']['access_token'] and config['twitter']['access_token_secret']):
|
if(config['twitter']['api_key'] and config['twitter']['api_key_secret'] and config['twitter']['access_token'] and config['twitter']['access_token_secret']):
|
||||||
twitter_client = authenticate_twitter(config['twitter'])
|
twitter_client = authenticate_twitter(config['twitter'])
|
||||||
|
|
Reference in a new issue