<jupyter_start><jupyter_text>Twitter API  Import Statements<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt

import json
import glob
import os<jupyter_output><empty_output><jupyter_text>Helper Funtions<jupyter_code># Renvoie une dataframe avec le contenu de tous les fichiers .json du dossier passé en argument
def createDF(path_to_folder, test_set=False):
    
    os.chdir(path_to_folder)

    df_results = pd.DataFrame(columns=['contributors', 'coordinates', 'created_at', 'display_text_range',
           'entities', 'extended_tweet', 'favorite_count', 'favorited',
           'filter_level', 'geo', 'id', 'id_str', 'in_reply_to_screen_name',
           'in_reply_to_status_id', 'in_reply_to_status_id_str',
           'in_reply_to_user_id', 'in_reply_to_user_id_str', 'is_quote_status',
           'lang', 'matching_rules', 'place', 'possibly_sensitive', 'quote_count',
           'quoted_status', 'quoted_status_id', 'quoted_status_id_str',
           'quoted_status_permalink', 'reply_count', 'retweet_count', 'retweeted',
           'retweeted_status', 'source', 'text', 'truncated', 'user'])
        
    for filename in glob.glob("*.json"):
        data = json.load(open(filename))
        df_current = pd.DataFrame(data["results"])
        df_results = pd.concat([df_results, df_current], sort=False)
    
    if test_set:
        df_results = df_results.sample(10)
    
    return df_results
# Ajoute une colonne date à partir de la colonne created_at
map_month = {'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 'Aug':8,'Sep':9, 'Oct':10, 'Nov':11, 'Dec':12,}

def extractDate(df):
    # création d'une colonne date avec les premiers caractères de la colonne created_at 
    df['date'] = df.created_at.str[:10]
    
    # division de la colonne date en un tuple new = weekday, month, date
    new = df['date'].str.split(' ', n = 2, expand = True)
    
    # ajout des colonnes day, month, year
    df['day'] = new[2] 
    df['month'] = new[1] 
    df.month = df.month.map(map_month)
    df['year'] = '2020'
    
    # modification de la colonne date pour avoir le format yyyy-mm-dd
    df['date'] = df['year'].str.cat(df[['month', 'day']].astype(str), sep="-")
    df['date'] = df['date'].astype('datetime64')
    
    # suppression des colonnes inutiles
    del df['day']
    del df['month']
    del df['year']
# Fonction d'affichage d'un histogramme pour une dataframe avec une colonne date (stack overflow)
def visualize(df, column_name='date', color='#494949', title=''):
    """
    Visualize a dataframe with a date column.

    Parameters
    ----------
    df : Pandas dataframe
    column_name : str
        Column to visualize
    color : str
    title : str
    """
    plt.figure(figsize=(20, 10))
    ax = (df[column_name].groupby([df[column_name].dt.month,df[column_name].dt.day])
                         .count()).plot(kind="bar", color=color)
    ax.set_facecolor('#ffffff')
    ax.set_xlabel("jour")
    ax.set_ylabel("nombre de tweets")
    ax.set_title(title)
    plt.show()<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>df_5G = createDF('/Users/benji/Desktop/GI01/IC05/Projet/Essaiv3/5G')
df_5G.shape
df_Gates = createDF('/Users/benji/Desktop/GI01/IC05/Projet/Essaiv3/Gates')
df_Gates.shape
df_hydro = createDF('/Users/benji/Desktop/GI01/IC05/Projet/Essaiv3/Hydroxychloroquine/')
df_hydro.shape<jupyter_output><empty_output><jupyter_text>Building a test dataset<jupyter_code>df_test5G = createDF('/Users/benji/Desktop/GI01/IC05/Projet/Essaiv3/Hydroxychloroquine/', test_set=True)
df_copy = df_test5G.copy(deep=True)
df_copy.head(2)
extractDate(df_copy)
df_copy['date']
plot_hist(df_copy)<jupyter_output><empty_output><jupyter_text>Testing on real dataset  Bill Gates Dataframe<jupyter_code>extractDate(df_Gates)

# On ne garde que les tweets des derniers mois
df_Gates = df_Gates[(df_Gates['date'] > '2020-10-05') & (df_Gates['date'] < '2020-12-07')]
visualize(df_Gates, color='#CD7058', title='Évolution du nombre de tweets relatifs à Gates et au coronavirus')<jupyter_output><empty_output><jupyter_text>5G Dataframe<jupyter_code>extractDate(df_5G)

# On ne garde que les tweets des derniers mois
df_5G = df_5G[(df_5G['date'] > '2020-10-05') & (df_5G['date'] < '2020-12-07')]
visualize(df_5G, color='#F9A65A', title='Évolution du nombre de tweets relatifs à la 5G et au coronavirus')<jupyter_output><empty_output><jupyter_text>Hydroxycloroquine Dataframe<jupyter_code>extractDate(df_hydro)

# On ne garde que les tweets des derniers mois
df_hydro = df_hydro[(df_hydro['date'] > '2020-10-05') & (df_hydro['date'] < '2020-12-07')]
visualize(df_hydro, color='#599AD3', title='Évolution du nombre de tweets relatifs à l\'hydroxychloroquine et au coronavirus')
df_hydro['text'].sample(10)<jupyter_output><empty_output><jupyter_text>Les 3 théories en même temps<jupyter_code>plt.figure(figsize=(20, 10))

# <NAME>
freq_gates = df_Gates['date'].value_counts()
plt.bar(freq_gates.index, freq_gates.values, color='lightcoral')

# Hydroxychloroquine
freq_hydro = df_hydro['date'].value_counts()
plt.bar(freq_hydro.index, freq_hydro.values, color='yellowgreen')

# 5G
freq_5g = df_5G['date'].value_counts()
plt.bar(freq_5g.index, freq_5g.values, color='lightskyblue')

plt.title('Évolution du nombre de tweets pour les 3 théories')

colors = {'Hydroxychloroquine':'yellowgreen', '5G':'lightskyblue', 'Gates':'lightcoral'}         
labels = list(colors.keys())
handles = [plt.Rectangle((0,0),1,1, color=colors[label]) for label in labels]
plt.legend(handles, labels)

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1475909009493104, 'fraction_numerical': 0.026688160487193264, 'mean_word_length': 4.8655462184873945, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '99226', 'license': 'MIT', 'n_tokens_mistral': 2192, 'n_tokens_neox': 2047, 'n_words': 399, 'path': 'scripts/01_building_df_barplot.ipynb', 'repo_name': 'baptisteviera/Data_Analysis_Of_Covid_19_Conspiracy_Theories', 'size': '5587'}