<jupyter_start><jupyter_text><jupyter_code>!pip install fire 
!pip install preprocessor
from requests import get
from requests.exceptions import RequestException
from contextlib import closing
from bs4 import BeautifulSoup
import pandas as pd
import os, sys

import fire

import csv
import sys
import os
import json
import matplotlib.pyplot as plt
import re
import string

import matplotlib.dates as mdates
import seaborn as sns

# to view all columns
pd.set_option("display.max.columns", None)

import tweepy
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy import API
from tweepy import Cursor
from datetime import datetime, date, time, timedelta
import sys


import preprocessor as p

auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api =tweepy.API(auth, wait_on_rate_limit=True,
          wait_on_rate_limit_notify=True)
class Tweets():
    
    # class variables + tweepy authentication
    def __init__(self, cols=None, auth=None,intervention_date = '2020-04-01', earliest_date = '2020-01-01'):
      if cols is None:
        self.cols =  ['created_at', 'source', 'original_text', 
                     'lang',
                    'favorite_count', 'retweet_count', 'original_author'
                    ,'name', "following", 'followers',
                    'hashtags', 'mention', 'place_coord_boundaries', 'location']
      else:
        self.cols = cols

      if auth is None:
        consumer_key="RPFdqfW02zjJBUz4own9u1bbq"
        consumer_secret="<KEY>"
        access_token="<KEY>"
        access_token_secret="<KEY>"
        auth = OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_token, access_token_secret)
      self.auth = auth
      self.api = tweepy.API(auth, wait_on_rate_limit=True,
          wait_on_rate_limit_notify=True)

     # self.intervention_date = intervention_date
       #self.end_date = datetime(2020, 4, 1, 0, 0)
      #self.earliest_date = earliest_date
      self.keywords = ['food shortage']

    # country id 
    def get_placeid(self, city):
        places = api.geo_search(query=city, granularity="city")
        place_id = places[0].id
        
        return place_id
    

    # get tweets
    def get_tweets(self, state, start_date ):
      place_id = get_placeid(state)
        # tweets before covid intervention
      tweets = tweepy.Cursor(api.search , q=keywords and ("place:%s" % place_id),  since_id = start_date).items(100)
      return tweets#, post_intervention_tweets, 
    
    # save tweets to dataframe and then csv
    def get_var(self, state, start_date, csvfile=None):

      df = pd.DataFrame(columns=self.cols)
      

      tweeted = get_tweets(state, start_date)  
      newentry = [] 
        for tweets in tweeted:
          new_entry += [tweets['created_at'],
                        tweets['source'], tweets['text'],
                        tweets['lang'],tweets['favorite_count'], 
                        tweets['retweet_count'], 
                        tweets['user']['screen_name'], tweets['user']['name'],
                        tweets['user']['friends_count'], tweets['user']['followers_count']]

          hashtags = ", ".join([hashtag_item['text'] for hashtag_item in status['entities']['hashtags']])


                #
          mentions = ", ".join([mention['screen_name'] for mention in status['entities']['user_mentions']])
          new_entry.append(mentions) #append the user mentions

          try:
              xyz = status['place']['bounding_box']['coordinates']
              coordinates = [coord for loc in xyz for coord in loc]
          except TypeError:
              coordinates = None
          #
          new_entry.append(coordinates)

          try:
              location = status['user']['location']
          except TypeError:
              location = ''
          #
          new_entry.append(location)

          #now append a row to the dataframe
          single_tweet_df = pd.DataFrame([new_entry], columns=self.cols)
          df = df.append(single_tweet_df, ignore_index=True)

        #write to csv  
        if not csvfile is None:
            #save it to file
            df.to_csv(f'{csvfile}_{state}.csv', columns=self.cols, index=False, encoding="utf-8")
        return df

# instantiate class
t = Tweets(auth)

# get place ids (in a dict)
ids = {}
for state in ['Lagos']:  #country twitter codes. za is for south africa
    placeid = t.get_placeid(state)
    ids[state] = placeid
    
ids  
 state, tweets, start_date, csvfile=None)
state = 'Lagos'
start_date= datetime(2019, 6, 1, 0, 0)


tweets_file = '/content/drive/My Drive/Twitter Data/tweets'
df = t.get_var(state,start_date, csvfile=tweets_file)<jupyter_output><empty_output><jupyter_text>Uncomment your country and run code, might take a while.<jupyter_code># get tweets 
KE_pre_intervention_tweets, KE_post_intervention_tweets = t.get_tweets(ids['KE'])
# NG_pre_intervention_tweets, NG_post_intervention_tweets = get_tweets(ids['NG'])
# UG_pre_intervention_tweets, UG_post_intervention_tweets = get_tweets(ids['UG'])
# ZA_pre_intervention_tweets, ZA_post_intervention_tweets = get_tweets(ids['ZA'])
KE_pre_intervention_tweets
KE_post_intervention_tweets

# t = []
# for tweet in KE_pre_intervention_tweets:
#     t.append(tweet.created_at)
    
# t
# get csv files

# pre intervention
t.get_var('KE', KE_pre_intervention_tweets, period = 'pre_covid')

# post intervention
t.get_var('KE', KE_post_intervention_tweets, period = 'post_covid')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10362047440699126, 'fraction_numerical': 0.007668985197075085, 'mean_word_length': 3.0994152046783627, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '28119', 'license': 'MIT', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1754, 'n_words': 406, 'path': 'Starter_code_for_scrap.ipynb', 'repo_name': 'NabeelSeid/Food-Quantity-Project', 'size': '5737'}