<jupyter_start><jupyter_text>Installed Python Libraries now import them<jupyter_code>import pymongo
from pymongo import MongoClient
import json
import tweepy
import twitter
from pprint import pprint
import configparser
import pandas as pd<jupyter_output><empty_output><jupyter_text>Load Authorization Information<jupyter_code>config = configparser.ConfigParser()
config.read('config.ini')

CONSUMER_KEY      = config['mytwitter']['api_key']
CONSUMER_SECRET   = config['mytwitter']['api_secrete']
OAUTH_TOKEN       = config['mytwitter']['access_token']
OATH_TOKEN_SECRET = config['mytwitter']['access_secrete']

mongod_connect = config['mymongo']['connection']<jupyter_output><empty_output><jupyter_text>Connect to MongoDB Cluster<jupyter_code>client = MongoClient(mongod_connect)
db = client.demo # use or create a database named demo
tweet_collection = db.tweet_collection #use or create a collection named tweet_collection
tweet_collection.create_index([("id", pymongo.ASCENDING)],unique = True) # make sure the collected tweets are unique<jupyter_output><empty_output><jupyter_text>Use the Rest API to Collect Tweets Authorization<jupyter_code>rest_auth = twitter.oauth.OAuth(OAUTH_TOKEN,OATH_TOKEN_SECRET,CONSUMER_KEY,CONSUMER_SECRET)
rest_api = twitter.Twitter(auth=rest_auth)<jupyter_output><empty_output><jupyter_text>Define query for rest API<jupyter_code>count = 100 #number of returned tweets, default and max is 100
geocode = "38.4392897,-78.9412224,50mi"  # defin the location, in Harrisonburg, VA
q = "election"                               #define the keywords, tweets contain election<jupyter_output><empty_output><jupyter_text>The retained tweets will contain election and be located in harrisonburg<jupyter_code>search_results = rest_api.search.tweets( count=count,q=q, geocode=geocode) #you can use both q and geocode
statuses = search_results["statuses"]
since_id_new = statuses[-1]['id']
for statuse in statuses:
    try:
        tweet_collection.insert_one(statuse)
        pprint(statuse['created_at'])# print the date of the collected tweets
    except:
        pass<jupyter_output>'Tue Nov 02 19:42:13 +0000 2021'
'Tue Nov 02 19:37:15 +0000 2021'
'Tue Nov 02 19:35:47 +0000 2021'
'Tue Nov 02 19:32:12 +0000 2021'
'Tue Nov 02 19:32:10 +0000 2021'
'Tue Nov 02 19:30:32 +0000 2021'
'Tue Nov 02 19:30:00 +0000 2021'
'Tue Nov 02 19:27:38 +0000 2021'
'Tue Nov 02 19:17:19 +0000 2021'
'Tue Nov 02 19:12:16 +0000 2021'
'Tue Nov 02 19:03:15 +0000 2021'
'Tue Nov 02 18:49:27 +0000 2021'
'Tue Nov 02 18:49:27 +0000 2021'
'Tue Nov 02 18:44:51 +0000 2021'
'Tue Nov 02 18:42:59 +0000 2021'
'Tue Nov 02 18:38:57 +0000 2021'
'Tue Nov 02 18:38:01 +0000 2021'
'Tue Nov 02 18:35:28 +0000 2021'
'Tue Nov 02 18:33:48 +0000 2021'
'Tue Nov 02 18:31:06 +0000 2021'
'Tue Nov 02 18:30:28 +0000 2021'
'Tue Nov 02 18:26:18 +0000 2021'
'Tue Nov 02 18:24:35 +0000 2021'
'Tue Nov 02 18:12:19 +0000 2021'
'Tue Nov 02 18:07:27 +0000 2021'
'Tue Nov 02 18:05:11 +0000 2021'
'Tue Nov 02 18:02:30 +0000 2021'
'Tue Nov 02 18:00:54 +0000 2021'
'Tue Nov 02 17:53:10 +0000 2021'
'Tue Nov 02 17:50:08 +0000 2021'
'Tue Nov 0[...]
{'fraction_non_alphanumeric': 0.1128254580520733, 'fraction_numerical': 0.16329154612664737, 'mean_word_length': 5.854625550660793, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '362367', 'license': 'MIT', 'n_tokens_mistral': 1559, 'n_tokens_neox': 1108, 'n_words': 335, 'path': 'Lec9.ipynb', 'repo_name': 'DyMarti/ia340', 'size': '3111'}