<jupyter_start><jupyter_text>Data Collectionhttps://developer.twitter.com/en/docs/twitter-api/v1/tweets/search/api-reference/get-search-tweetsIn this library we use the Twitter standard search API, which returns a collection of relevant Tweets matching a specified query.The Search API is not meant to be an exhaustive source of Tweets. **Not all Tweets will be indexed or made available via the search interface**.https://developer.twitter.com/en/docs/twitter-api/v1/tweets/timelines/guides/working-with-timelinesThe Twitter API has several methods, such as GET statuses/user_timeline and GET statuses/home_timeline, which return a timeline of Tweet data. Such timelines can grow very large, so there are limits to how much of a timeline a client application may fetch in a single request. Applications must therefore iterate through timeline results in order to build a more complete list.<jupyter_code>########################### Data Collection Functions #####################

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import nltk
import re
import seaborn as sns
from twython import Twython
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from sklearn.decomposition import LatentDirichletAllocation



ACCESS_TOKEN = "<KEY>"
ACCESS_TOKEN_SECRET = "<KEY>"
API_KEY = "<KEY>"
API_SECRET_KEY = "<KEY>"

STOP_WORDS = ["'d", "'ll", "'re", "'s", "'ve", 'could', 'might', 'must',
              "n't", 'need', 'sha', 'wo', 'would', "ca", "na", "rt", "like",
             'u', 'get', 'got']

def collect_tweets(query='', geocode=None, result_type='recent',
                   num_of_page=20, count=100, since=None, until=None):
    '''Collects a number of tweets using Twitter standard search API and 
    returns a list of dictionaries each representing a tweet.

    query: search query
    geocode: Returns tweets by users located within a given radius 
             of the given lat/long. The parameter value is specified 
             by " latitude,longitude,radius "
    result_type: Specifies what type of search results you would prefer to receive. 
                  mixed : Include both popular and real time results in the response.
                  recent : return only the most recent results in the response
                  popular : return only the most popular results in the response.
    num_of_page: number of pages to collect.
    count: The number of tweets to return per page, up to a maximum of 100. 
           Defaults to 15.
    since: Returns tweets created after the given date. 
           Date should be formatted as YYYY-MM-DD. 
           The search index has a 7-day limit.
    until: Returns tweets created before the given date. 
           Date should be formatted as YYYY-MM-DD. 
           The search index has a 7-day limit.
    since_id: Returns results with an ID greater than 
              (that is, more recent than) the specified ID. 
              There are limits to the number of Tweets which 
              can be accessed through the API. If the limit of 
              Tweets has occured since the since_id, the since_id 
              will be forced to the oldest ID available.
    max_id: Returns results with an ID less than 
            (that is, older than) or equal to the specified ID.
    include_entities: The entities node will not be included when set to false.
    '''

    # Authentication
    twitter_obj = Twython(API_KEY, API_SECRET_KEY,
                          ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

    # Use Twitter standard API search
    tweet_result = twitter_obj.search(q=query, geocode=geocode,
                                      result_type=result_type, count=count,
                                      since=since, until=until,
                                      include_entities='true',
                                      tweet_mode='extended', lang='en')

    # In order to prevent redundant tweets explained here
    # https://developer.twitter.com/en/docs/tweets/timelines/guides/working-with-timelines
    # instead of reading a timeline relative to the top of the list
    # (which changes frequently), an application should read the timeline
    # relative to the IDs of tweets it has already processed.
    tweets_list = tweet_result['statuses'] 
    i = 0  # num of iteration through each page
    rate_limit = 1  # There is a limit of 100 API calls in the hour
    while tweet_result['statuses'] and i < num_of_page:
        if rate_limit < 1:
            # Rate limit time out needs to be added here in order to
            # collect data exceeding available rate-limit
            print(str(rate_limit)+' Rate limit!')
            break
        max_id = tweet_result['statuses'][len(
            tweet_result['statuses']) - 1]['id']-1

        tweet_result_per_page = twitter_obj.search(q=query, geocode=geocode,
                                                   result_type=result_type,
                                                   count=count, since=since,
                                                   until=until,
                                                   include_entities='true',
                                                   tweet_mode='extended',
                                                   lang='en',
                                                   max_id=str(max_id))

        tweets_list += tweet_result_per_page['statuses']
        i += 1
        rate_limit = int(twitter_obj.get_lastfunction_header(
            'x-rate-limit-remaining'))

    return tweets_list

def find_hashtags(tweet):
    hashtags = ''
    for i, term in enumerate(tweet):
        hashtags += term['text']+','
    return hashtags
    

def make_dataframe(tweets_list, search_term):
    '''Gets the list of tweets and return it as a pandas DataFrame.
    '''

    df = pd.DataFrame()
    df['tweet_id'] = list(map(lambda tweet: tweet['id'],
                              tweets_list))
    df['user'] = list(map(lambda tweet: tweet['user']
                          ['screen_name'], tweets_list))
    df['time'] = list(map(lambda tweet: tweet['created_at'], tweets_list))
    df['tweet_text'] = list(map(lambda tweet: tweet['full_text'], tweets_list))
    df['location'] = list(
        map(lambda tweet: tweet['user']['location'], tweets_list))
    hashtag = ''
    df['hashtags'] = list(
        map(lambda tweet: find_hashtags(tweet['entities']['hashtags']), tweets_list))
    df['search_term'] = list(map(lambda tweet: search_term if search_term.lower(
    ) in tweet['full_text'].lower() else None, tweets_list))

    return df



############################ Data Cleaning Functions ####################################

def clean_tweet_text(tweet, user_flag=True, urls_flag=True, 
                     punc_flag=True, number_flag=True,
                     special_char_flag=True,
                     stop_word_flag=False):
    '''Clean a tweet by performing the following.

    - Remove username
    - Remove urls
    - Remove all punctuation and special character
    - Remove all stopwords if flag is True
    - Returns a cleaned text
    '''

    # remove the user
    if user_flag:
        tweet = re.sub(r'@[w\w]+', ' ', tweet)

    # remove the urls
    if urls_flag:
        tweet = re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b', ' ', tweet)

    # replace the negations
    tweet = re.sub(r"n't", ' not', tweet)
    tweet = re.sub(r"N'T", ' NOT', tweet)

    # remove punctuations
    if punc_flag:
        tweet = re.sub('[!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@’…”'']+', ' ', tweet)

    # remove numbers
    if number_flag:
        tweet = re.sub('([0-9]+)', '', tweet)

    # remove special characters
    if special_char_flag:
        tweet = re.sub(r'[^\w]', ' ', tweet)

    # remove double space
    tweet = re.sub('\s+', ' ', tweet)

    if stop_word_flag:
        tweet = ' '.join([word for word in tweet.split() if word.lower(
        ) not in stopwords.words('english')+STOP_WORDS])

    return tweet



############# NLP #############3

def tokenize_tweet(tweet):
    '''Convert the normal text strings in to a list of tokens(words)
    '''
    # Tokenization
    return [word for word in tweet.split()]

    
def create_bag_of_words(tweets, max_df=1.0, min_df=1,
                       max_features=None):
    '''Vectorize the tweets using bag of words.
    
    Return the vectorized/bag of words of tweets
    as well as the features' name.
    
    max_df: float in range [0.0, 1.0] or int, default=1.0
            When building the vocabulary ignore terms that 
            have a document frequency strictly higher than 
            the given threshold (corpus-specific stop words). 
            If float, the parameter represents a proportion 
            of documents, integer absolute counts. 
            This parameter is ignored if vocabulary is not None.

    min_df: float in range [0.0, 1.0] or int, default=1
            When building the vocabulary ignore terms that 
            have a document frequency strictly lower than 
            the given threshold. This value is also called 
            cut-off in the literature. If float, the parameter 
            represents a proportion of documents, integer absolute 
            counts. This parameter is ignored if vocabulary is not None.
    '''
    
    # Vectorization using Countvectorize
    cv = CountVectorizer(analyzer=tokenize_tweet, max_df=max_df, 
                         min_df=min_df, max_features=max_features)
    tweets_bow = cv.fit_transform(tweets)
    feature_names = cv.get_feature_names()
    return tweets_bow, feature_names

def create_tfidf(tweets_bow):
    '''Create the TF-IDF of tweets
    '''
    tfidf_transformer = TfidfTransformer().fit(tweets_bow)
    tweets_tfidf = tfidf_transformer.transform(tweets_bow)
    return tweets_tfidf
    

def sentiment_analysis(tweet):
    '''Takes a tweet and return a dictionary of scores in 4 categories.
    - Negative score
    - Neutral score
    - Positive score
    - Compound score
    
    Special characters and stopwords need to stay in the tweet.
    '''
    
    #create an instance of SentimentIntensityAnalyzer
    sid = SentimentIntensityAnalyzer()
    return sid.polarity_scores(tweet)



#sklearn.decomposition.LatentDirichletAllocation
def topic_modeling(tweets, num_components=7, 
                   num_words_per_topic=10, 
                   random_state=42,
                   max_df=1.0, min_df=1,
                   max_features=None):
    '''Get all the tweets and return the topics
    and the highest probability words per topic
    '''
    
    #create the bags of words
    tweets_bow, feature_names = create_bag_of_words(tweets, max_df=max_df, 
                                                    min_df=min_df, 
                                                    max_features=max_features)
    
    #create an instace of LatentDirichletAllocation 
    lda = LatentDirichletAllocation(n_components=num_components,
                                    random_state=random_state)
    lda.fit(tweets_bow)
    
    #grab the highest probability words per topic
    words_per_topic = {}
    for index,topic in enumerate(lda.components_):
        words_per_topic[index] = [feature_names[i] for i in topic.argsort()[-15:]]
        
    topic_results = lda.transform(tweets_bow)
    topics = topic_results.argmax(axis=1)
    return topics, words_per_topic
    
    
def find_top_hashtags(hashtags):
    '''Get the hashtag column and find the popular hashtags.
    
    '''
    pass
    
    
    
#test

query = 'Walmart'
#geocode="43.653226,-79.383184,100km" #Toronto
#geocode="49.525238,-93.874023,4000km" #North America
tweets_list = collect_tweets(query=query, geocode="49.525238,-93.874023,4000km")

df = make_dataframe(tweets_list, query)
df.info()
pd.set_option('display.max_rows', 999)
pd.set_option('display.max_columns', 999)
df.head(400)
df.to_csv('tweets.csv')
df = pd.read_csv('tweets.csv')
df['clean_text'] = df['tweet_text'].apply(lambda text: clean_tweet_text(text, punc_flag=False, 
                                                                        number_flag=False, special_char_flag=False,))
df['no_stop_words_text'] = df['tweet_text'].apply(lambda text: clean_tweet_text(text, stop_word_flag=True))
df['sentiment_scores'] = df['clean_text'].apply(lambda tweet: sentiment_analysis(tweet))
df['compound_scores'] = df['sentiment_scores'].apply(lambda scores_dict: scores_dict['compound'])
df['topics'], words_per_topic = topic_modeling(df['no_stop_words_text'], num_components=7, 
                   num_words_per_topic=10, 
                   random_state=42,
                   max_df=0.7, min_df=5,
                   max_features=250)
words_per_topic
#pd.set_option('display.max_rows', 999)
pd.set_option('display.max_columns', 999)
df.head(400)
df.loc[1]['tweet_text']
df.loc[341]['clean_text']
df.loc[341]['no_stop_words_text']
df.loc[341]['hashtags']
df.loc[1]['sentiment_scores']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10369750456482045, 'fraction_numerical': 0.011488131466828972, 'mean_word_length': 2.489514202282984, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 10, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '258102', 'license': 'FTL', 'n_tokens_mistral': 4005, 'n_tokens_neox': 3723, 'n_words': 1112, 'path': 'notebooks/1.0-sara-initial-data-collection.ipynb', 'repo_name': '2020ai/tweetDetective_DataScience', 'size': '13294'}