<jupyter_start><jupyter_text>Congress twitter analysisThis notebook is created for the Mamba python course as an example for big data analysis. developed by MAMBA  table of content:1. [import files](1)2. [read twitter data](2)3. [read user and party data](3)4. [combine datasets](4)5. [analyze twitter data per party](5)6. [plot results](6)  1. import files<jupyter_code>import pandas as pd
#You most likely do not have textblob in you anaconda. 
#If not, go to your anaconda prompt and type pip install textblob (enter)
#The package should install and the ModuleNotFoundError should dissapear
import textblob as tb
import matplotlib.pyplot as plt
import matplotlib
import datetime as dt
import pickle
import numpy as np
#settings (seaborn for style of graphs and maplotlib inline to let the graphs show inside this notebook)
%matplotlib inline
plt.style.use('seaborn')<jupyter_output><empty_output><jupyter_text>2. read twitter data<jupyter_code>start_date = dt.datetime(2018,2,1)
end_date   = dt.datetime(2018,2,3)
date_range = pd.date_range(start_date, end_date)
twitter_data = pd.DataFrame()
for datum in date_range:
    #The twitter data of congress men and women are stored in a file on github. 
    #twitter_data_day saves this data in a pandas dataframe
    twitter_data_day = pd.read_json('https://raw.githubusercontent.com/alexlitel/congresstweets/master/data/%s.json'%datum.strftime('%Y-%m-%d'))
    #the time is set as index of the dataframe
    twitter_data_day.index = pd.to_datetime(twitter_data_day.time)
    #for each day the twitter frame of that day is concatenated with the main twitter_data dataframe
    twitter_data = pd.concat([twitter_data, twitter_data_day])<jupyter_output><empty_output><jupyter_text>3. read user and political party data<jupyter_code>#Which twitter user belongs to which political party
user_twitter = pd.read_json('https://raw.githubusercontent.com/alexlitel/congresstweets-automator/master/data/historical-users-filtered.json')
user_twitter['user_id'] = user_twitter.accounts.apply(lambda x: int(x[0]['id']))
user_twitter['screen_name'] = user_twitter.accounts.apply(lambda x: x[0]['screen_name'])<jupyter_output><empty_output><jupyter_text>4. combine datasetslink twitter users to political parties<jupyter_code>#Make a new column with the political party per twitter message in twitter_data
twitter_data['party'] = twitter_data.user_id.apply(lambda x: user_twitter[user_twitter.user_id==x]['party'].values[0] if x in user_twitter.user_id.values else 'U')<jupyter_output><empty_output><jupyter_text>5. analyze twitter data per partyanalyze the sentiment of the tweets<jupyter_code>#textblob is a packages that has a function to evaluate the sentiment of that message. 
#This sentiment.polarity is valued as a float between -1 and 1. 
twitter_data['polarity'] = twitter_data.text.apply(lambda x: tb.Sentence(x).sentiment.polarity)
#make a new dataframe with only relevant data columns
twitter_data_rel = twitter_data[['screen_name','party', 'polarity']]
# load pickle to skip previous time consuming steps

with open(r'data\processed_twitter_data.pickle', 'rb') as pfile:
    twitter_data_rel = pickle.load(pfile)
# get daily average of the polarity per party (decomcrat or republican)
twitter_data_d = twitter_data_rel[twitter_data_rel.party=='D']
twitter_data_r = twitter_data_rel[twitter_data_rel.party=='R']

#Note, in de lines below the 'D' stands for daily instead of 'democrat'. 
#This does not improve readibility, but ah well. 
twitter_data_d_daily_average = twitter_data_d.resample('D').mean()
twitter_data_r_daily_average = twitter_data_r.resample('D').mean()
# get bandwidth 0f the polarity per party daily ('D') (decomcrat or republican)
upper_percentile = 90
lower_percentile = 10

#With apply we do the operation per row of echt dataframe
twitter_data_d_daily_upper = twitter_data_d.resample('D').apply(lambda x: np.percentile(x['polarity'], q=upper_percentile))
twitter_data_d_daily_lower = twitter_data_d.resample('D').apply(lambda x: np.percentile(x['polarity'], q=lower_percentile))
twitter_data_r_daily_upper = twitter_data_r.resample('D').apply(lambda x: np.percentile(x['polarity'], q=upper_percentile))
twitter_data_r_daily_lower = twitter_data_r.resample('D').apply(lambda x: np.percentile(x['polarity'], q=lower_percentile))<jupyter_output><empty_output><jupyter_text>6 plot results<jupyter_code>#First lets get some statistics 
df = twitter_data_rel.groupby(['screen_name']).mean()

low_ave_pol = df.polarity.min()
user_low_ave_pol = df[df.polarity==low_ave_pol].index[0]
no_tweets_user = (twitter_data_rel.screen_name==user_low_ave_pol).sum()
print('minimum')
print('lowest, average polarity of twitter users: %.2f'%low_ave_pol)
print('twitter user with lowest average polarity: %s'%user_low_ave_pol)
print("number of tweets by %s : %i"%(user_low_ave_pol, no_tweets_user))
print('\nmaximum')
max_ave_pol = df.polarity.max()
user_max_ave_pol = df[df.polarity==max_ave_pol].index[0]
no_tweets_user = (twitter_data_rel.screen_name==user_max_ave_pol).sum()
print('maximum, average polarity of twitter users: %.2f'%max_ave_pol)
print('twitter user with maximum average polarity: %s'%user_max_ave_pol)
print("number of tweets by %s : %i"%(user_max_ave_pol, no_tweets_user))
# annotation of these values in our plot
def add_annotation(ax, day, twitter_data_daily_average, 
                   connectionstyle, text, x_afstand=-20, y_afstand=0.3, facecolor='green', linewidth=None,
                   edgecolor=None):
    '''
    '''
    pol_day_party = twitter_data_daily_average[day:day]['polarity'].values[0]
    ax.annotate(text, 
                xy=(day, pol_day_party), 
                xytext=(day+dt.timedelta(x_afstand), pol_day_party+y_afstand), 
                fontsize=12,
                arrowprops=dict(arrowstyle="fancy", 
                                facecolor=facecolor, 
                                edgecolor=edgecolor ,
                                lw=linewidth,
                                connectionstyle=connectionstyle))
    return ax
twitter_data_rel.loc['party_mod',:] = twitter_data_rel['party']
twitter_data_rel['party_mod', ~twitter_data_rel['party_mod'].isin(['R','D'])] = 'Other'
twitter_data_rel.loc['party_mod', twitter_data_rel['party_mod']=='R'] = 'Republican'
twitter_data_rel.loc['party_mod', twitter_data_rel['party_mod']=='D'] = 'Democrat'

plt.rcParams.update({'font.size': 16})
#plt.xticks(rotation=45)

fig, axs = plt.subplots(1, 2, figsize=(10,4))


chart_1 = twitter_data_rel.groupby([twitter_data_rel.party_mod]).count()
total = sum(chart_1['party'])
ax1, ax2  = axs[0], axs[1]
# ax3, ax4 = axs[1, 0], axs[1, 1]
ax1.pie(chart_1['party'].values, labels=chart_1['party'].index,  
        autopct=lambda p: '{:,.0f}'.format(p * total / 100), #explode=[ 0.05, 0.05, 0.05] ,
        shadow=True, startangle=90, textprops={'fontsize': 14})

ax1.set_title('number of tweets', fontsize=18)
ax1.axis('equal')

chart_2 = twitter_data_rel.groupby(['party_mod']).mean()
chart_2 = chart_2.reindex(['Democrat', 'Republican', 'Other'])
chart_2.loc['Total']= twitter_data_rel.polarity.mean()


ax2.bar([ 0, 1, 2, 3], chart_2.values[:,0])

ax2.set_xticks(np.arange(-0, 4, 1))
ax2.set_xticklabels(chart_2.index.values, rotation=90)


ax2.tick_params(axis='x', which='major', labelsize=16)

ax2.set_yticks(np.arange(0,0.25,0.05))
ax2.tick_params(axis='y', which='major', labelsize=12)

ax2.patch.set_facecolor('none')
ax2.set_title('average sentiment', fontsize=18);
plt.subplots_adjust(wspace=0.3)
fig.savefig(r'figures/pie_bar_charts.png', bbox_inches='tight')

chart_2
# plot
fig = plt.figure(figsize=(16,16/2.75))
ax = plt.subplot()

# plot average
twitter_data_d_daily_average.polarity.plot(ax=ax, linestyle=None, marker='.', color='blue', label='Democrats')
twitter_data_r_daily_average.polarity.plot(ax=ax, linestyle=None, marker='.', color='red', label='Republicans')

# plot bandwidth
ax.fill_between(twitter_data_d_daily_upper.index, twitter_data_d_daily_lower.values, 
                twitter_data_d_daily_upper.values, facecolor='blue', alpha=0.1)
ax.fill_between(twitter_data_r_daily_upper.index, twitter_data_r_daily_lower.values, 
                twitter_data_r_daily_upper.values, facecolor='red', alpha=0.1)


# add arrow to some peaks
ax = add_annotation(ax, dt.datetime(2018,4,1), twitter_data_r_daily_average, "angle3,angleA=90,angleB=0",
                    'Easter', x_afstand=-12)

ax = add_annotation(ax, dt.datetime(2018,5,13), twitter_data_r_daily_average, "arc3,rad=-0.3",
                    "mother's day", x_afstand=2, facecolor='blue')

ax = add_annotation(ax, dt.datetime(2018,6,17), twitter_data_r_daily_average, "arc3,rad=-0.3",
                    "father's day", x_afstand=-2, facecolor='magenta')

ax = add_annotation(ax, dt.datetime(2018,6,18), twitter_data_d_daily_average, "arc3,rad=-0.2",
                    "refugee family separation crisis", x_afstand=-30, y_afstand=-0.3, facecolor='orange')

ax = add_annotation(ax, dt.datetime(2018,7,4), twitter_data_r_daily_average, "arc3,rad=-0.2",
                    "independence day", x_afstand=2 , facecolor='red', edgecolor='blue', linewidth=1)

# format axes
ax.set_ylim(-0.5,1.0)
ax.set_xlabel('')
ax.set_ylabel('sentiment of tweets', fontsize=12)
ax.set_yticklabels([])

ax.set_xlim(start_date, end_date)
xmin, xmax = ax.get_xlim()
ymin, ymax = ax.get_ylim()

ax.text(xmin-5, ymax-0.04, r'positive $\longrightarrow$', rotation=90, fontsize=12)
ax.text(xmin-5, ymin+0.30, r'$\longleftarrow$ negative', rotation=90, fontsize=12)

# format legend
box = ax.get_position()
ax.set_position([box.x0, box.y0 + box.height * 0.1,
                 box.width, box.height * 0.9])


# Put a legend below current axis
leg = ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.08),
                frameon=True, facecolor='white', edgecolor='black', shadow=True, ncol=5)

ax.set_title('Twitter analysis US congress')

fig.savefig(r'figures\twitter_analysis_US_congress.png', dpi=300, bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13257462314066087, 'fraction_numerical': 0.02425876010781671, 'mean_word_length': 5.40537084398977, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '144409', 'license': 'MIT', 'n_tokens_mistral': 3714, 'n_tokens_neox': 3461, 'n_words': 725, 'path': 'Practical_examples/data_analysis/congress_twitter_analysis/congress_twitter_analysis.ipynb', 'repo_name': 'MAMBA-python/raw_material', 'size': '10017'}