<jupyter_start><jupyter_text>Assignment 04  Note: To be done after completing Module 6In this exercise, we will work on some problems to get practical experience on the concepts learnt so far.  Problem StatementAs a class project, we will write a client program which will extract tweets with the hash-tag 'happy'. This client will handle rate-limit errors and collect as many tweets as possible in a given time.The tweets collected would be stored as a CSV file `"tweets.csv"` with the following contents: `id`, `lang`, `user.screen_name`, `text`.<jupyter_code># create functions for handling tweets
import csv
import tweepy
import time
import os
import sys

modulePath = os.path.abspath(os.path.join("../../.."))
if modulePath not in sys.path:
    sys.path.append(modulePath)
import config  # this is a file on my local machine that stores the API keys and access tokens

# Helper functions
# implement the ratelimiting function
def limit_handler(cursor):
    while True:
        try:
            yield cursor.next()
        except tweepy.TweepError:
            print("Sleeping for 15 minutes")
            time.sleep(15 * 60)


# ------------------------------------------------------------------------
# Main Program
# ------------------------------------------------------------------------

# paste you consumer keys and access tokens here
consumer_key = config.apiKey
consumer_secret = config.apiSec
access_token = config.accessToken
access_token_secret = config.accessSec

# create api object after proper authentication
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

search_tag = "#happy"
fname = "tweets.csv"

# creating a search cursor
search_cursor = tweepy.Cursor(api.search, search_tag).items(5000)

with open(fname, "w") as f:
    field_names = ["ID", "Language", "Screen Name", "Text"]
    for tweet in limit_handler(search_cursor):
        exportwriter = csv.writer(f)
        exportwriter.writerow(
            [tweet.id, tweet.lang, tweet.user.screen_name, tweet.text.encode("utf-8")]
        )
print("Write to file successful!")
# Load first 10 tweets to review results
fname = "tweets.csv"
with open(fname) as f:
    counter = 0
    filereader = csv.reader(f)
    for row in filereader:
        if counter == 10:
            break
        else:
            print("****************")
            print(row)
            counter += 1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14834276475343572, 'fraction_numerical': 0.008084074373484237, 'mean_word_length': 2.767123287671233, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '261707', 'license': 'MIT', 'n_tokens_mistral': 712, 'n_tokens_neox': 672, 'n_words': 242, 'path': 'Assignment-4.ipynb', 'repo_name': 'petersontylerd/web-scraping-course', 'size': '2474'}