Tweet echoer implemented.
This commit is contained in:
Alex Huddleston
parent
de2be0b7af
commit
c7a90f1e80
4 changed files with 114 additions and 96 deletions
BIN
__pycache__/tweet_converter.cpython-37.pyc
Normal file
BIN
__pycache__/tweet_converter.cpython-37.pyc
Normal file
Binary file not shown.
41
tweet_converter.py
Normal file
41
tweet_converter.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
# TODO: Clean this up and write comments.
|
||||
def convert_tweets(tweets):
|
||||
|
||||
recent_tweets = tweets
|
||||
|
||||
outtweets = []
|
||||
|
||||
for tweet in recent_tweets:
|
||||
|
||||
try:
|
||||
outtweets.insert(0, {
|
||||
'id':str(tweet.id_str),
|
||||
'date':str(tweet.created_at),
|
||||
'retweet':('RT @' in tweet.text),
|
||||
'text':str(tweet.text.encode('utf-8'))[1:],
|
||||
'hashtags':tweet.entities.get('hashtags'),
|
||||
'mentions':tweet.entities.get('user_metions')
|
||||
})
|
||||
|
||||
except:
|
||||
try:
|
||||
outtweets.insert(0, {
|
||||
'id':str(tweet.id_str),
|
||||
'date':str(tweet.created_at),
|
||||
'retweet':('RT @' in tweet.full_text),
|
||||
'text':str(tweet.full_text.encode('utf-8'))[1:],
|
||||
'hashtags':tweet.entities.get('hashtags'),
|
||||
'mentions':tweet.entities.get('user_metions')
|
||||
})
|
||||
|
||||
except:
|
||||
outtweets.insert(0, {
|
||||
'id':str(tweet.id_str),
|
||||
'date':str(tweet.created_at),
|
||||
'retweet':True,
|
||||
'text':str(tweet.retweeted_status.text.encode("utf-8"))[1:],
|
||||
'hashtags':tweet.entities.get('hashtags'),
|
||||
'mentions':tweet.entities.get('user_metions')
|
||||
})
|
||||
|
||||
return outtweets
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# Some legacy code I wrote for a project in college. It's ugly and messy. I'll clean it up and repurpose it later.
|
||||
|
||||
import tweepy #https://github.com/tweepy/tweepy
|
||||
import json
|
||||
from time import sleep
|
||||
|
||||
|
||||
def get_all_tweets(screen_name):
|
||||
#Twitter only allows access to a users most recent 3240 tweets with this method
|
||||
with open('keys.txt', 'r') as f:
|
||||
#Twitter API credential
|
||||
consumer_key = f.readline().rstrip()
|
||||
consumer_secret = f.readline().rstrip()
|
||||
access_key = f.readline().rstrip()
|
||||
access_secret = f.readline().rstrip()
|
||||
|
||||
#authorize twitter, initialize tweepy
|
||||
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
|
||||
auth.set_access_token(access_key, access_secret)
|
||||
api = tweepy.API(auth)
|
||||
|
||||
#initialize a list to hold all the tweepy Tweets
|
||||
recent_tweets = []
|
||||
|
||||
#make initial request for most recent tweets (200 is the maximum allowed count)
|
||||
new_tweets = api.user_timeline(screen_name = screen_name,count=10,tweet_mode = 'extended')
|
||||
|
||||
#save most recent tweets
|
||||
recent_tweets.extend(new_tweets)
|
||||
'''
|
||||
|
||||
#save the id of the oldest tweet less one
|
||||
oldest = alltweets[-1].id - 1
|
||||
|
||||
#keep grabbing tweets until there are no tweets left to grab
|
||||
while len(new_tweets) > 0 and len(alltweets) < 3000:
|
||||
print("getting tweets before %s" % (oldest))
|
||||
|
||||
#all subsiquent requests use the max_id param to prevent duplicates
|
||||
new_tweets = api.user_timeline(screen_name = screen_name,count=200,max_id=oldest)
|
||||
|
||||
#save most recent tweets
|
||||
alltweets.extend(new_tweets)
|
||||
|
||||
#update the id of the oldest tweet less one
|
||||
oldest = alltweets[-1].id - 1
|
||||
|
||||
print("...%s tweets downloaded so far" % (len(alltweets)))
|
||||
|
||||
'''
|
||||
#transform the tweepy tweets into a 2D array that will populate the csv
|
||||
outtweets = []
|
||||
for tweet in recent_tweets:
|
||||
try:
|
||||
if tweet.retweeted or ('RT @' in tweet.text):
|
||||
outtweets.append([tweet.id_str, tweet.created_at, "True", tweet.retweeted_status.text.encode("utf-8"), tweet.entities.get('hashtags'), tweet.entities.get('user_mentions')])
|
||||
except:
|
||||
try:
|
||||
outtweets.append([tweet.id_str, tweet.created_at, ('RT @' in tweet.full_text), tweet.full_text.encode("utf-8"), tweet.entities.get('hashtags'), tweet.entities.get('user_mentions')])
|
||||
except:
|
||||
outtweets.append([tweet.id_str, tweet.created_at, ('RT @' in tweet.text), tweet.text.encode("utf-8"), tweet.entities.get('hashtags'), tweet.entities.get('user_mentions')])
|
||||
|
||||
for t in outtweets:
|
||||
print(t[3])
|
||||
|
||||
'''
|
||||
#write the csv
|
||||
with open('%s_tweets.csv' % screen_name, 'w') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(["id","created_at","retweet","text","hashtags","user_mentions"])
|
||||
writer.writerows(outtweets)
|
||||
|
||||
pass
|
||||
'''
|
||||
|
||||
if __name__ == '__main__':
|
||||
with open('users.txt', 'r') as f:
|
||||
for l in f:
|
||||
get_all_tweets(l.rstrip())
|
||||
|
||||
'''
|
||||
|
||||
if __name__ == '__main__':
|
||||
#pass in the username of the account you want to download
|
||||
counter = 0
|
||||
with open("users.txt", "r") as f:
|
||||
for line in f:
|
||||
get_all_tweets(line.rstrip())
|
||||
counter = counter + 1
|
||||
if (counter % 5) == 0:
|
||||
sleep(15*60)
|
||||
|
||||
'''
|
||||
73
tweet_echoer.py
Normal file
73
tweet_echoer.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# Tweet echoing works. I just need this to be refactored into something more useful...
|
||||
from tweet_converter import convert_tweets
|
||||
from time import sleep
|
||||
import tweepy
|
||||
|
||||
# Tentative... Might just settle with line-by-line with encryption
|
||||
# TODO: Import and utilize encryption methods.
|
||||
import json
|
||||
|
||||
# TODO: Major clean up.
|
||||
def echo_recent_tweets(screen_name):
|
||||
|
||||
# TODO: Encrypt this file for extra security?
|
||||
# TODO: Probably move all of this to its own function.
|
||||
with open('keys.txt', 'r') as f:
|
||||
#Twitter API credential
|
||||
consumer_key = f.readline().rstrip()
|
||||
consumer_secret = f.readline().rstrip()
|
||||
access_key = f.readline().rstrip()
|
||||
access_secret = f.readline().rstrip()
|
||||
|
||||
# Authorize twitter, initialize tweepy
|
||||
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
|
||||
auth.set_access_token(access_key, access_secret)
|
||||
api = tweepy.API(auth)
|
||||
|
||||
last_tweet = 0
|
||||
|
||||
# TODO: Don't do while(True). Use another safer method.
|
||||
while(True):
|
||||
|
||||
# Request the last 10 tweets, since I'm assuming I won't have made 10 tweets in 30 seconds...
|
||||
new_tweets = api.user_timeline(screen_name = screen_name,count=10,tweet_mode = 'extended')
|
||||
|
||||
recent_tweets = []
|
||||
|
||||
# Only bother processing new tweets.
|
||||
for tweet in new_tweets:
|
||||
if(tweet.id > last_tweet):
|
||||
recent_tweets.append(tweet)
|
||||
|
||||
# Only do this if there are new tweets.
|
||||
if recent_tweets:
|
||||
outtweets = convert_tweets(recent_tweets)
|
||||
last_tweet = int(outtweets[len(outtweets) - 1]['id'])
|
||||
|
||||
# Debugging purposes...
|
||||
# TODO: Clean this up probably.
|
||||
for t in outtweets:
|
||||
print(t['date'] + '\t' + t['text'])
|
||||
|
||||
sleep(30)
|
||||
|
||||
# TODO: Stop reading users from a file, unnecessary.
|
||||
if __name__ == '__main__':
|
||||
with open('users.txt', 'r') as f:
|
||||
for l in f:
|
||||
echo_recent_tweets(l.rstrip())
|
||||
|
||||
# Clean these up lmao
|
||||
|
||||
'''
|
||||
#write the csv
|
||||
with open('%s_tweets.csv' % screen_name, 'w') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(["id","created_at","retweet","text","hashtags","user_mentions"])
|
||||
writer.writerows(outtweets)
|
||||
|
||||
pass
|
||||
'''
|
||||
Reference in a new issue