<jupyter_start><jupyter_text><NAME>'s Approval---<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sqlalchemy import create_engine

from datetime import datetime, timedelta

from statsmodels.stats.weightstats import DescrStatsW

import sys

import os
path = os.getcwd()
path = path[:path.find('brazil_election2022')] + 'brazil_election2022'

sys.path.append(path + '\\src\\data')
import make_dataset<jupyter_output><empty_output><jupyter_text>---------  1. collecting Twitter data Example: collection tweets from <NAME> (key attributes)The following example exemplifies how to use Twitter api and it main attributes.To simplify the example, we show tweets from <NAME>, only. The process to capture general tweets addressing <NAME> somehow is time demanding and it will not be displayed on this notebook.<jupyter_code>!python ..\\src\\data\\Twitter\\0-brazil.cursor_timeline.py
path = 'mysql+pymysql://root:root@localhost:3306/'
database = 'brazil_election'
table = 'tweets_from'
engine = create_engine(path+database)

tweets = pd.read_sql_table(table, con=engine)\
            .set_index(['id', 'tweet_id']).dropna(axis='columns', how='all')
tweets<jupyter_output><empty_output><jupyter_text>---------  2. collecting poll data<jupyter_code>approval_RawData = make_dataset.presidentApproval_download()
approval_RawData.head()
approval_RawData['PRESIDENTE'].unique()<jupyter_output><empty_output><jupyter_text>interest: <NAME> data<jupyter_code>jb_approval_RawData = approval_RawData[approval_RawData['PRESIDENTE']=='<NAME>']
jb_approval_RawData.head()<jupyter_output><empty_output><jupyter_text>Our focus relies on understanding <NAME> approval over time. Different polls evaluate different aspects of the government. To minimize the risk of adding noise to our model, we will focus on polls that evaluate approval over 4 pillars (positive, negative, regular, dont know/didnt answered)<jupyter_code>cols = ['POSITIVA', 'REGULAR', 'NEGATIVA', 'NS.NR']
jb_approval_RawData = jb_approval_RawData.loc[jb_approval_RawData[cols].dropna(axis='rows', how='any').index]

jb_approval_RawData.head()
jb_approval_RawData[cols] = jb_approval_RawData[cols].replace({' ':np.nan, '':np.nan})
jb_approval_RawData[cols] = jb_approval_RawData[cols].astype(float)
jb_approval_RawData = jb_approval_RawData[~jb_approval_RawData['EMPRESA'].isna()]
jb_approval_RawData = jb_approval_RawData[jb_approval_RawData['TIPO'] == 'Avaliação do governo federal']
jb_approval_RawData = jb_approval_RawData[jb_approval_RawData['UFs'].fillna(0).astype(str).apply(lambda x: x.isnumeric())]
cols = ['positiva', 'regular', 'negativa', 'ns.nr']
colors_dict = {'positiva':'darkgreen', 'regular':'darkorange', 'negativa':'darkred', 'ns.nr':'black'}

plt.figure(figsize=(12.5, 7.5))
for col in cols:   
    plt.scatter(x=jb_approval_RawData.index, y=jb_approval_RawData[col.upper()], 
                color=colors_dict[col], s=2.5)
    
plt.title('<NAME> - approval', fontsize=12.5)    
plt.xticks(rotation=-45, ha='left')
#plt.legend(loc=2)
plt.grid(ls='--', alpha=0.5)
plt.show()<jupyter_output><empty_output><jupyter_text>For calculating the average over time, we expect to weight each poll by a pollster rating (qualitative information) and by sample size of each poll. Therefore, we fullfill missing values of `N` feature (sample size) with the mean value of the same pollster, only then with the mean value of the dataset, if information not available.<jupyter_code>for presidente in jb_approval_RawData['PRESIDENTE'].unique():
        df = jb_approval_RawData[jb_approval_RawData['PRESIDENTE'] == presidente]
        for empresa in df['EMPRESA'].unique():
            mean_n = df[df['EMPRESA'] == empresa]['N'].mean()
            if pd.isna(mean_n):
                mean_n = df['N'].mean()
            if pd.isna(mean_n):
                mean_n = jb_approval_RawData['N'].mean()
            mean_n = int(mean_n)
            loc_filter = (jb_approval_RawData['EMPRESA'] == empresa)&(jb_approval_RawData['PRESIDENTE'] == presidente)
            jb_approval_RawData.loc[loc_filter, 'N'] = jb_approval_RawData.loc[loc_filter, 'N'].fillna(mean_n)
jb_approval_RawData.head()<jupyter_output><empty_output><jupyter_text>---------  3. modelling<jupyter_code>make_dataset.presidentApproval_preprocessig()
path = os.getcwd()
path = path[:path.find('brazil_election2022')] + 'brazil_election2022'
approval = pd.read_csv(path+'\\data\\processed\\presidentApproval_processed.csv', parse_dates=['datafim'])
approval.head()
bolsonaro_approval = approval[approval['presidente'] == '<NAME>']
bolsonaro_approval = bolsonaro_approval[bolsonaro_approval['ufs'].apply(lambda x: x.isnumeric())]
bolsonaro_approval = bolsonaro_approval.set_index('datafim').sort_index()
bolsonaro_approval.head()<jupyter_output><empty_output><jupyter_text>---  getting weighted average<jupyter_code>def get_stats(data, k):
    """
    return main statistics of approval series (weighted avg and weighted std)
    input:
        - data (dataframe): df with ['positiva', 'regular', 'negativa', 'ns.nr'] series
        - k (int): last k polls to consider
    output:
        - dataframe with avg and std    
    """
    mean = []
    std = []
    index_dates = pd.date_range(start='2019-02-01', end=datetime.today())
    
    for day in index_dates:
        df = data.loc[:day.date()].tail(k)

        descriptive_stats = DescrStatsW(df[['positiva', 'regular', 'negativa', 'ns.nr']].astype(float), weights=df['peso'])

        mean.append(descriptive_stats.mean)
        std.append(descriptive_stats.std)

    mean = pd.DataFrame(mean, index=index_dates, columns=['positiva_avg', 'regular_avg', 'negativa_avg', 'ns.nr_avg'])
    std = pd.DataFrame(std, index=index_dates, columns=['positiva_std', 'regular_std', 'negativa_std', 'ns.nr_std'])
    
    return mean.join(std)
stats_k05 = get_stats(bolsonaro_approval, k=5)
stats_k10 = get_stats(bolsonaro_approval, k=10)
stats_k10.tail()<jupyter_output><empty_output><jupyter_text>1. how popular is Bolsonaro’s approval changing over time?<jupyter_code>cols = ['positiva', 'regular', 'negativa', 'ns.nr']
colors_dict = {'positiva':'darkgreen', 'regular':'darkorange', 'negativa':'darkred', 'ns.nr':'black'}

plt.figure(figsize=(12.5, 7.5))
for col in cols:
    col_avg = col+'_avg'
    col_std = col+'_std'
    plt.plot(stats_k10[col_avg], color=colors_dict[col], label=col)
    plt.fill_between(x=stats_k10.index, color= colors_dict[col], alpha=0.25,
                     y2=stats_k10[col_avg]-stats_k10[col_std], 
                     y1=stats_k10[col_avg]+stats_k10[col_std])
    
    plt.scatter(x=bolsonaro_approval.index, y=bolsonaro_approval[col], 
                color=colors_dict[col], s=2.5)

    
plt.title('<NAME> - aprovação', fontsize=12.5)    
plt.xticks(rotation=-45, ha='left')
plt.legend(loc=2)
plt.grid(ls='--', alpha=0.5)
plt.savefig(path+'\\reports\\figures\\JairBolsonaro_approval.png', bbox_inches='tight')
plt.show()
cols = ['empresa', 'n', 'positiva', 'regular', 'negativa', 'ns.nr']
cols_rename = {'positiva':'positive', 'negativa':'negative', 'ns.nr':'don\'t know'}

print('last 5 polls:')
bolsonaro_approval[cols].tail(5).reset_index().set_index(['datafim', 'empresa', 'n'])[::-1].rename(cols_rename, axis='columns')<jupyter_output>last 5 polls:<jupyter_text>---------  Engagement Index - social media The Engagement Index intends to capture the <NAME> attraction on Twitter. As the index and approval share similar trends, the index reveals itself as a prominent feature to predict approval in short term.For future studies and analysis (new ideas), I decided to not share the modeling involved.<jupyter_code>def get_social_media_index():
    """
    expertise
    """
    pass

engag_index = pd.read_csv(path+'\\data\\external\\EngagementIndex.csv', sep=';', parse_dates=True, index_col=0)<jupyter_output><empty_output><jupyter_text>2. how Bolsonaro’s social media is changing over time?<jupyter_code>plt.figure(figsize=(12.5, 7.5))

plt.plot(engag_index, color='black', label='Engagement Index')

plt.title('<NAME> - Engagement Index', fontsize=12.5)    
plt.xticks(rotation=-45, ha='left')
plt.legend(loc=1)
plt.grid(ls='--', alpha=0.5)
plt.savefig(path+'\\reports\\figures\\JairBolsonaro_EngagementIndex.png', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>---------  Comparing Engagement Index to Jair Bolsonaro approval rate  3. could a social media engagement be a precursor variable of Bolsonaro’s approval?<jupyter_code>fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(12.5, 10))
ax3 = ax2.twinx()

ax1.plot(stats_k10['positiva_avg'], color='darkgreen', label='positiva')
ax1.plot(stats_k10['regular_avg'], color='darkorange', label='regular')
ax2.plot(np.sum(stats_k10[['positiva_avg', 'regular_avg']], axis='columns'), color='blue', label='positiva + regular')
ax3.plot(engag_index, color='black', label='Engagement Index')


ax1.set_title('<NAME> - approval', fontsize=12.5)    
plt.xticks(rotation=-45, ha='left')
fig.legend(loc='upper right', bbox_to_anchor=(0.795, 0.775))
ax1.grid(ls='--', alpha=0.5)
ax2.grid(ls='--', alpha=0.5)
plt.savefig(path+'\\reports\\figures\\JairBolsonaro_EngagementIndex+Approval.png', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>---------<jupyter_code>from subprocess import call
call(['python', '-m', 'nbconvert', '1.1-gbf-2020brazil_bolsonaroApproval.ipynb']);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15485647706810718, 'fraction_numerical': 0.016735515305582035, 'mean_word_length': 5.566063977746871, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '564190', 'license': 'MIT', 'n_tokens_mistral': 3479, 'n_tokens_neox': 3302, 'n_words': 630, 'path': 'notebooks/1.1-gbf-2020brazil_bolsonaroApproval.ipynb', 'repo_name': 'gabrielbfs/brazil_election2020', 'size': '9545'}