<jupyter_start><jupyter_text>Collect Tweets into MongoDB  Install Python librariesYou may need to restart your Jupyter Notebook instance after installed those libraries.<jupyter_code>!pip install pymongo
!pip install pymongo[srv]
!pip install dnspython
!pip install tweepy
!pip install twitter<jupyter_output>Requirement already satisfied: twitter in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (1.18.0)
[33mWARNING: You are using pip version 20.0.2; however, version 20.2.4 is available.
You should consider upgrading via the '/home/ec2-user/anaconda3/envs/python3/bin/python -m pip install --upgrade pip' command.[0m<jupyter_text>Import Python libraries<jupyter_code>import pymongo
from pymongo import MongoClient
import json
import tweepy
import twitter
from pprint import pprint
import configparser
import pandas as pd<jupyter_output><empty_output><jupyter_text>Load the Authorization Info Save database connection info and API Keys in a config.ini file and use the configparse to load the authorization info.<jupyter_code>config = configparser.ConfigParser()
config.read('config.ini')

CONSUMER_KEY      = config['mytwitter']['api_key']
CONSUMER_SECRET   = config['mytwitter']['api_secret']
OAUTH_TOKEN       = config['mytwitter']['access_token']
OATH_TOKEN_SECRET = config['mytwitter']['access_secret']

mongod_connect = config['mymongo']['connection']<jupyter_output><empty_output><jupyter_text>Connect to the MongoDB Cluster<jupyter_code>client = MongoClient(mongod_connect)
db = client.gp3 # use or create a database named demo
tweet_collection = db.tweet_collection #use or create a collection named tweet_collection
tweet_collection.create_index([("id", pymongo.ASCENDING)],unique = True) # make sure the collected tweets are unique<jupyter_output><empty_output><jupyter_text>Use the Streaming API to Collect Tweets Authorize the Stream API<jupyter_code>stream_auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
stream_auth.set_access_token(OAUTH_TOKEN, OATH_TOKEN_SECRET)

strem_api = tweepy.API(stream_auth)<jupyter_output><empty_output><jupyter_text>Define the query for the Stream API<jupyter_code>track = ['election'] # define the keywords, tweets contain election

locations = [-78.9326449,38.4150904,-78.8816972,38.4450731] #defin the location, in Harrisonburg, VA<jupyter_output><empty_output><jupyter_text>The collected tweets will contain 'election'  OR  are located in Harrisonburg, VA<jupyter_code>class MyStreamListener(tweepy.StreamListener):
    def on_status(self, status):
        print (status.id_str)
        try:
            tweet_collection.insert_one(status._json)
        except:
            pass
  
    def on_error(self, status_code):
        if status_code == 420:
            #returning False in on_data disconnects the stream
            return False
myStreamListener = MyStreamListener()
myStream = tweepy.Stream(auth = strem_api.auth, listener=myStreamListener)
myStream.filter(track=track)#  (locations = locations)   #Use either track or locations<jupyter_output>1330653900412694534
1330653900328857603
1330653900664418311
1330653900135948292
1330653900681007104
1330653900857282567
1330653901025112064
1330653900878311431
1330653901482168321
1330653901431902208
1330653901574516736
1330653902086303751
1330653902069506050
1330653902224695297
1330653902417620998
1330653902383894528
1330653902635667457
1330653902698590210
1330653902572822531
1330653903344451584
1330653903642365959
1330653903717797896
1330653903726252036
1330653903864651781
1330653904007270402
1330653904288112640
1330653904611151872
1330653905202638849
1330653905286541316
1330653905378742272
1330653905500450816
1330653905466896385
1330653905655566338
1330653905907310592
1330653906053918720
1330653906469175297
1330653906515451905
1330653906700005379
1330653906855202819
1330653906741981185
1330653906905522176
1330653907140263936
1330653907673108488
1330653907807158272
1330653908046213120
1330653908025405441
1330653908050579457
1330653908289675267
1330653908063186947
1330653908448931840
[...]<jupyter_text>Use the REST API to Collect Tweets Authorize the REST API<jupyter_code>rest_auth = twitter.oauth.OAuth(OAUTH_TOKEN,OATH_TOKEN_SECRET,CONSUMER_KEY,CONSUMER_SECRET)
rest_api = twitter.Twitter(auth=rest_auth)<jupyter_output><empty_output><jupyter_text>Define the query for the REST API<jupyter_code>count = 100 #number of returned tweets, default and max is 100
geocode = "38.4392897,-78.9412224,50mi"  # defin the location, in Harrisonburg, VA
q = "election"                               #define the keywords, tweets contain election<jupyter_output><empty_output><jupyter_text>The collected tweets will contain 'election'  AND  are located in Harrisonburg, VA<jupyter_code>search_results = rest_api.search.tweets( count=count,q=q, geocode=geocode) #you can use both q and geocode
statuses = search_results["statuses"]
since_id_new = statuses[-1]['id']
for statuse in statuses:
    try:
        tweet_collection.insert_one(statuse)
        pprint(statuse['created_at'])# print the date of the collected tweets
    except:
        pass<jupyter_output>'Sun Nov 22 23:25:53 +0000 2020'
'Sun Nov 22 23:20:06 +0000 2020'
'Sun Nov 22 23:17:41 +0000 2020'
'Sun Nov 22 23:12:02 +0000 2020'
'Sun Nov 22 23:10:04 +0000 2020'
'Sun Nov 22 23:05:14 +0000 2020'
'Sun Nov 22 23:00:54 +0000 2020'
'Sun Nov 22 22:56:01 +0000 2020'
'Sun Nov 22 22:53:20 +0000 2020'
'Sun Nov 22 22:49:08 +0000 2020'
'Sun Nov 22 22:46:29 +0000 2020'
'Sun Nov 22 22:45:25 +0000 2020'
'Sun Nov 22 22:41:19 +0000 2020'
'Sun Nov 22 22:36:55 +0000 2020'
'Sun Nov 22 22:36:47 +0000 2020'
'Sun Nov 22 22:35:03 +0000 2020'
'Sun Nov 22 22:31:13 +0000 2020'
'Sun Nov 22 22:28:56 +0000 2020'
'Sun Nov 22 22:24:45 +0000 2020'
'Sun Nov 22 22:17:55 +0000 2020'
'Sun Nov 22 21:57:34 +0000 2020'
'Sun Nov 22 21:54:58 +0000 2020'
'Sun Nov 22 21:42:07 +0000 2020'
'Sun Nov 22 21:39:42 +0000 2020'
'Sun Nov 22 21:29:17 +0000 2020'
'Sun Nov 22 21:28:59 +0000 2020'
'Sun Nov 22 21:28:41 +0000 2020'
'Sun Nov 22 21:28:24 +0000 2020'
'Sun Nov 22 21:28:11 +0000 2020'
'Sun Nov 22 21:27:54 +0000 2020'
'Sun Nov 2[...]<jupyter_text>Continue fetching early tweets with the same query. YOU WILL REACH YOUR RATE LIMIT VERY FAST<jupyter_code>since_id_old = 0
while(since_id_new != since_id_old):
    since_id_old = since_id_new
    search_results = rest_api.search.tweets( count=count,q=q,
                        geocode=geocode, max_id= since_id_new)
    statuses = search_results["statuses"]
    since_id_new = statuses[-1]['id']
    for statuse in statuses:
        try:
            tweet_collection.insert_one(statuse)
            pprint(statuse['created_at']) # print the date of the collected tweets
        except:
            pass<jupyter_output>'Sun Nov 22 15:55:06 +0000 2020'
'Sun Nov 22 15:54:14 +0000 2020'
'Sun Nov 22 15:53:08 +0000 2020'
'Sun Nov 22 15:49:29 +0000 2020'
'Sun Nov 22 15:47:56 +0000 2020'
'Sun Nov 22 15:46:58 +0000 2020'
'Sun Nov 22 15:37:03 +0000 2020'
'Sun Nov 22 15:33:27 +0000 2020'
'Sun Nov 22 15:30:35 +0000 2020'
'Sun Nov 22 15:29:37 +0000 2020'
'Sun Nov 22 15:25:08 +0000 2020'
'Sun Nov 22 15:23:28 +0000 2020'
'Sun Nov 22 15:21:11 +0000 2020'
'Sun Nov 22 15:14:28 +0000 2020'
'Sun Nov 22 15:14:05 +0000 2020'
'Sun Nov 22 15:12:41 +0000 2020'
'Sun Nov 22 15:10:50 +0000 2020'
'Sun Nov 22 15:08:47 +0000 2020'
'Sun Nov 22 15:07:04 +0000 2020'
'Sun Nov 22 15:04:48 +0000 2020'
'Sun Nov 22 15:01:04 +0000 2020'
'Sun Nov 22 14:59:11 +0000 2020'
'Sun Nov 22 14:57:59 +0000 2020'
'Sun Nov 22 14:55:19 +0000 2020'
'Sun Nov 22 14:55:06 +0000 2020'
'Sun Nov 22 14:54:49 +0000 2020'
'Sun Nov 22 14:49:09 +0000 2020'
'Sun Nov 22 14:48:31 +0000 2020'
'Sun Nov 22 14:46:40 +0000 2020'
'Sun Nov 22 14:45:14 +0000 2020'
'Sun Nov 2[...]<jupyter_text>View the Collected Tweets Print the number of tweets and unique twitter users<jupyter_code>print(tweet_collection.estimated_document_count())# number of tweets collected

user_cursor = tweet_collection.distinct("user.id")
print (len(user_cursor)) # number of unique Twitter users<jupyter_output>9410
8041<jupyter_text>Create a text index and print the Tweets containing specific keywords.<jupyter_code>tweet_collection.create_index([("text", pymongo.TEXT)], name='text_index', default_language='english') # create a text index<jupyter_output><empty_output><jupyter_text>Create a cursor to query tweets with the created index<jupyter_code>tweet_cursor = tweet_collection.find({"$text": {"$search": "vote"}}) # return tweets contain vote<jupyter_output><empty_output><jupyter_text>Use pprint to display tweets<jupyter_code>for document in tweet_cursor[0:10]: # display the first 10 tweets from the query
    try:
        print ('----')
#         pprint (document) # use pprint to print the entire tweet document
   
        print ('name:', document["user"]["name"]) # user name
        print ('text:', document["text"])         # tweets
    except:
        print ("***error in encoding")
        pass
tweet_cursor = tweet_collection.find({"$text": {"$search": "vote"}}) # return tweets contain vote<jupyter_output><empty_output><jupyter_text>Use pandas to display tweets<jupyter_code>tweet_df = pd.DataFrame(list(tweet_cursor ))
tweet_df[:10] #display the first 10 tweets
tweet_df["favorite_count"].hist() # create a histogram show the favorite count<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09797622871827819, 'fraction_numerical': 0.21640432594496198, 'mean_word_length': 5.6335227272727275, 'pattern_counts': {'":': 4, '<': 60, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '40180', 'license': 'MIT', 'n_tokens_mistral': 4751, 'n_tokens_neox': 3337, 'n_words': 916, 'path': 'lab11 (1).ipynb', 'repo_name': 'baileyga/ia340', 'size': '9339'}