<jupyter_start><jupyter_text>Collect Tweets into MongoDB  Install Python librariesYou may need to restart your Jupyter Notebook instance after installed those libraries.<jupyter_code>!pip install pymongo
!pip install pymongo[srv]
!pip install dnspython
!pip install tweepy
!pip install twitter<jupyter_output>Requirement already satisfied: twitter in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (1.18.0)
[33mWARNING: You are using pip version 20.0.2; however, version 20.3.3 is available.
You should consider upgrading via the '/home/ec2-user/anaconda3/envs/python3/bin/python -m pip install --upgrade pip' command.[0m<jupyter_text>Import Python libraries<jupyter_code>import pymongo
from pymongo import MongoClient
import json
import tweepy
import twitter
from pprint import pprint
import configparser
import pandas as pd<jupyter_output><empty_output><jupyter_text>Load the Authorization Info Save database connection info and API Keys in a config.ini file and use the configparse to load the authorization info.<jupyter_code>config = configparser.ConfigParser()
config.read('config.ini')

CONSUMER_KEY      = config['mytwitter']['api_key']
CONSUMER_SECRET   = config['mytwitter']['api_secrete']
OAUTH_TOKEN       = config['mytwitter']['access_token']
OATH_TOKEN_SECRET = config['mytwitter']['access_secrete']

mongod_connect = config['mymongo']['connection']<jupyter_output><empty_output><jupyter_text>Connect to the MongoDB Cluster<jupyter_code>client = MongoClient(mongod_connect)
db = client.gp2 # use or create a database named demo
Final_project = db.Final_project 
Final_project.create_index([("id", pymongo.ASCENDING)],unique = True) # make sure the collected tweets are unique<jupyter_output><empty_output><jupyter_text>Use the Streaming API to Collect Tweets Authorize the Stream API<jupyter_code>stream_auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
stream_auth.set_access_token(OAUTH_TOKEN, OATH_TOKEN_SECRET)

strem_api = tweepy.API(stream_auth)<jupyter_output><empty_output><jupyter_text>Define the query for the Stream API<jupyter_code>track = ['election'] # define the keywords, tweets contain election

locations = [-78.9326449,38.4150904,-78.8816972,38.4450731] #defin the location, in Harrisonburg, VA<jupyter_output><empty_output><jupyter_text>The collected tweets will contain 'election'  OR  are located in Harrisonburg, VA<jupyter_code>class MyStreamListener(tweepy.StreamListener):
    def on_status(self, status):
        print (status.id_str)
        try:
            tweet_collection.insert_one(status._json)
        except:
            pass
  
    def on_error(self, status_code):
        if status_code == 420:
            #returning False in on_data disconnects the stream
            return False
myStreamListener = MyStreamListener()
myStream = tweepy.Stream(auth = strem_api.auth, listener=myStreamListener)
myStream.filter(track=track)#  (locations = locations)   #Use either track or locations<jupyter_output><empty_output><jupyter_text>Use the REST API to Collect Tweets Authorize the REST API<jupyter_code>rest_auth = twitter.oauth.OAuth(OAUTH_TOKEN,OATH_TOKEN_SECRET,CONSUMER_KEY,CONSUMER_SECRET)
rest_api = twitter.Twitter(auth=rest_auth)<jupyter_output><empty_output><jupyter_text>Define the query for the REST API<jupyter_code>count = 100 #number of returned tweets, default and max is 100
geocode = "41.82399,-71.41283,100mi"  # defin the location, 
q = "COVID-19"                               #define the keywords,<jupyter_output><empty_output><jupyter_text>The collected tweets will contain 'election'  AND  are located in Harrisonburg, VA<jupyter_code>search_results = rest_api.search.tweets( count=count,q=q, geocode=geocode) #you can use both q and geocode
statuses = search_results["statuses"]
since_id_new = statuses[-1]['id']
for statuse in statuses:
    try:
        Final_project.insert_one(statuse)
        pprint(statuse['created_at'])# print the date of the collected tweets
    except:
        pass<jupyter_output>'Fri Dec 18 20:49:23 +0000 2020'
'Fri Dec 18 20:49:11 +0000 2020'
'Fri Dec 18 20:49:00 +0000 2020'
'Fri Dec 18 20:48:53 +0000 2020'
'Fri Dec 18 20:48:33 +0000 2020'
'Fri Dec 18 20:47:53 +0000 2020'
'Fri Dec 18 20:46:50 +0000 2020'
'Fri Dec 18 20:46:39 +0000 2020'
'Fri Dec 18 20:46:28 +0000 2020'
'Fri Dec 18 20:46:27 +0000 2020'
'Fri Dec 18 20:46:13 +0000 2020'
'Fri Dec 18 20:46:05 +0000 2020'
'Fri Dec 18 20:46:04 +0000 2020'
'Fri Dec 18 20:45:52 +0000 2020'
'Fri Dec 18 20:45:43 +0000 2020'
'Fri Dec 18 20:45:31 +0000 2020'
'Fri Dec 18 20:45:21 +0000 2020'
'Fri Dec 18 20:45:04 +0000 2020'
'Fri Dec 18 20:45:00 +0000 2020'
'Fri Dec 18 20:44:53 +0000 2020'
'Fri Dec 18 20:44:45 +0000 2020'
'Fri Dec 18 20:44:21 +0000 2020'
'Fri Dec 18 20:44:11 +0000 2020'
'Fri Dec 18 20:43:26 +0000 2020'
'Fri Dec 18 20:43:19 +0000 2020'
'Fri Dec 18 20:43:14 +0000 2020'
'Fri Dec 18 20:42:51 +0000 2020'
'Fri Dec 18 20:42:48 +0000 2020'
'Fri Dec 18 20:42:45 +0000 2020'
'Fri Dec 18 20:42:40 +0000 2020'
'Fri Dec 1[...]<jupyter_text>Continue fetching early tweets with the same query. YOU WILL REACH YOUR RATE LIMIT VERY FAST<jupyter_code>since_id_old = 0
while(since_id_new != since_id_old):
    since_id_old = since_id_new
    search_results = rest_api.search.tweets( count=count,q=q,
                        geocode=geocode, max_id= since_id_new)
    statuses = search_results["statuses"]
    since_id_new = statuses[-1]['id']
    for statuse in statuses:
        try:
            Final_project.insert_one(statuse)
            pprint(statuse['created_at']) # print the date of the collected tweets
        except:
            pass<jupyter_output>'Fri Dec 18 20:23:50 +0000 2020'
'Fri Dec 18 20:23:49 +0000 2020'
'Fri Dec 18 20:23:46 +0000 2020'
'Fri Dec 18 20:23:45 +0000 2020'
'Fri Dec 18 20:23:34 +0000 2020'
'Fri Dec 18 20:23:27 +0000 2020'
'Fri Dec 18 20:23:10 +0000 2020'
'Fri Dec 18 20:22:56 +0000 2020'
'Fri Dec 18 20:22:44 +0000 2020'
'Fri Dec 18 20:22:41 +0000 2020'
'Fri Dec 18 20:22:35 +0000 2020'
'Fri Dec 18 20:22:05 +0000 2020'
'Fri Dec 18 20:21:46 +0000 2020'
'Fri Dec 18 20:21:37 +0000 2020'
'Fri Dec 18 20:21:21 +0000 2020'
'Fri Dec 18 20:20:25 +0000 2020'
'Fri Dec 18 20:20:17 +0000 2020'
'Fri Dec 18 20:20:07 +0000 2020'
'Fri Dec 18 20:20:06 +0000 2020'
'Fri Dec 18 20:19:41 +0000 2020'
'Fri Dec 18 20:19:23 +0000 2020'
'Fri Dec 18 20:19:19 +0000 2020'
'Fri Dec 18 20:19:18 +0000 2020'
'Fri Dec 18 20:19:13 +0000 2020'
'Fri Dec 18 20:19:05 +0000 2020'
'Fri Dec 18 20:19:00 +0000 2020'
'Fri Dec 18 20:18:44 +0000 2020'
'Fri Dec 18 20:18:41 +0000 2020'
'Fri Dec 18 20:18:25 +0000 2020'
'Fri Dec 18 20:18:04 +0000 2020'
'Fri Dec 1[...]<jupyter_text>View the Collected Tweets Print the number of tweets and unique twitter users<jupyter_code>print(Final_project.estimated_document_count())# number of tweets collected

user_cursor = Final_project.distinct("user.id")
print (len(user_cursor)) # number of unique Twitter users<jupyter_output>8216
6161<jupyter_text>Create a text index and print the Tweets containing specific keywords.<jupyter_code>Final_project.create_index([("text", pymongo.TEXT)], name='text_index', default_language='english') # create a text index<jupyter_output><empty_output><jupyter_text>Create a cursor to query tweets with the created index<jupyter_code>tweet_cursor = Final_project.find({"$text": {"$search": "coronavirus"}}) # return tweets<jupyter_output><empty_output><jupyter_text>Use pprint to display tweets<jupyter_code>for document in tweet_cursor[0:10]: # display the first 10 tweets from the query
    try:
        print ('----')
#         pprint (document) # use pprint to print the entire tweet document
   
        print ('name:', document["user"]["name"]) # user name
        print ('text:', document["text"])         # tweets
    except:
        print ("***error in encoding")
        pass
tweet_cursor = Final_project.find({"$text": {"$search": "coronavirus"}}) # return tweets<jupyter_output><empty_output><jupyter_text>Use pandas to display tweets<jupyter_code>tweet_df = pd.DataFrame(list(tweet_cursor ))
tweet_df[:10] #display the first 10 tweets
tweet_df["favorite_count"].hist() # create a histogram show the favorite count<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11093521675596688, 'fraction_numerical': 0.13029712615684363, 'mean_word_length': 5.124533929903057, 'pattern_counts': {'":': 4, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '242301', 'license': 'MIT', 'n_tokens_mistral': 3789, 'n_tokens_neox': 2840, 'n_words': 849, 'path': 'Final.ipynb', 'repo_name': 'mcdanielva/ia340', 'size': '8212'}