<jupyter_start><jupyter_text>Time/volume tweets - line chart<jupyter_code>tendaguru_clean.columns
tendaguru_clean.date
tendaguru_clean.date[0].year
tendaguru_clean['year'] = tendaguru_clean['date'].apply(lambda x: x.year)
tendaguru_clean['year'].sample(5)<jupyter_output><empty_output><jupyter_text>Time/volume tweets - line chart<jupyter_code>tweetByYear = tendaguru_clean.groupby('year').count().reset_index()
tweetByYear
plt.figure(figsize=[17,4])
plt.xticks(rotation=45)
plt.title('Count of Twitter Conversations Per Year Scraped from Google Scholar: <NAME>')
g = sns.barplot(data=tweetByYear, 
             x="year", 
             y="content") 
ax=g
for p in ax.patches:
    ax.annotate("%.0f" % p.get_height(), (p.get_x() + p.get_width() / 2., p.get_height()),
                ha='center', va='center', fontsize=8, color='gray', xytext=(0, 5),
                textcoords='offset points')
_ = ax.set_ylim(0,200) #To make space for the annotations62222
plt.savefig('TwitterConvCount_tendaguru')<jupyter_output><empty_output><jupyter_text>Keywords - Network diagram<jupyter_code>tendaguru_clean[tendaguru_clean.clean_tweet == None]
def network_diagram(content_col, filt_val):
    """Draws a network diagram from a given text column
    
        Args:
            :param content_col: Series of strings, column we wish to get a network of.
            :param filt_val: integer, higher numbers filters only on words that have the 
            strongest connection
    """
    
    # You can define your own parameters
    cv = CountVectorizer(ngram_range=(1,1), stop_words = 'english') 
    X = cv.fit_transform(content_col.values.astype('U'))

    # This is the matrix manipulation step
    Xc = (X.T * X) 

    # This are the entity names (i.e. keywords)
    names = cv.get_feature_names() 
    
    # create dataframe of similar names
    df_co = pd.DataFrame(data = Xc.toarray(), columns = names, index = names)

    # put df in workable format
    a = df_co.stack().reset_index()
    a.columns = ['var1', 'var2','value']

    a_filtered=a.loc[ (a['value'] > filt_val) & (a['var1'] != a['var2']) ]


    # Build your graph
    plt.figure(figsize=(20,10))

    G=nx.from_pandas_edgelist(a_filtered, 'var1', 'var2')

    # Plot the network
    nx.draw(G, with_labels=True, node_color='orange', node_size=50, edge_color='green', linewidths=2, font_size=15)

network_diagram(tendaguru_clean.clean_tweet, 10)
plt.savefig('NetworkDiagram_tendaguru.png')<jupyter_output><empty_output><jupyter_text>People vs mentions - CSV list, bar chart<jupyter_code>import json
tendaguru_clean[['user','mentionedUsers']].sample(10)
def mentions(mentionedUsers):
    mentions = []
    try:
        mentions = []
        for d in mentionedUsers:
            mentions.append(d['username'])
    except TypeError:
        pass
    return mentions
    
pd.set_option('display.max_columns', None)  
pd.set_option('display.max_colwidth', -1)
tendaguru_clean['username'] = tendaguru_clean.user.apply(lambda x: x['username'])
tendaguru_clean['mentionedUsers'] = tendaguru_clean.mentionedUsers.apply(lambda x: mentions(x))
tendaguru_clean[['username', 'mentionedUsers']].to_csv('../data/peopleVmentions_tendaguru.csv')
from collections import Counter
mentionList = list(tendaguru_clean['mentionedUsers'])
flat_list = [item for sublist in mentionList for item in sublist]
flat_list[:10]
most_mentioned = dict(Counter(flat_list).most_common(20))
most_mentioned
plt.figure(figsize=[17,4])
plt.xticks(rotation=45)
plt.title('Most mentioned users')
g = plt.bar(most_mentioned.keys(), most_mentioned.values())
ax=g
plt.savefig('MostMentioned_tendaguru.png')<jupyter_output><empty_output><jupyter_text>Geography - bar chart<jupyter_code>tendaguru_clean.columns
tendaguru_clean.coordinates is None
loc_bb = tendaguru_clean[tendaguru_clean.coordinates.isnull() == False]
print(loc_bb.shape)
print(tendaguru_clean.shape)<jupyter_output>(6, 30)
(341, 30)<jupyter_text>For now, we will substitute with a simple bar graph with information about the countries in our dataset.<jupyter_code>loc_bb['long'] = loc_bb.coordinates.apply(lambda x:x['longitude'])
loc_bb['lat'] = loc_bb.coordinates.apply(lambda x:x['latitude'])
# import plotly.express as px

# fig = px.choropleth(df, geojson=counties, locations='fips', color='unemp',
#                            color_continuous_scale="Viridis",
#                            range_color=(0, 12),
#                            scope="usa",
#                            labels={'unemp':'unemployment rate'}
#                           )
# fig.update_layout(margin={"r":0,"t":0,"l":0,"b":0})
# fig.show()
tendaguru_clean['location'] = tendaguru_clean.user.apply(lambda x: x['location'])
has_loc = tendaguru_clean[tendaguru_clean.location != '']
has_loc.shape
s1 = has_loc[has_loc.location.str.contains(',')]
s1['country'] = s1.location.apply(lambda x: x.split(', ')[-1])
s1['no_words'] = s1.country.apply(lambda x: len(x.split()))
s1 = s1[s1.no_words == 1]
s1.shape
african_countries = ["Algeria",
"Angola",
"Benin",
"Botswana",
"Burkina Faso",
"Burundi",
"Cabo Verde",
"Cameroon",
"Central African Republic (CAR)",
"Chad",
"Comoros",
"Congo, Democratic Republic of the",
"Congo, Republic of the",
"Cote d'Ivoire",
"Djibouti",
"Egypt",
"Equatorial Guinea",
"Eritrea",
"Eswatini (formerly Swaziland)",
"Ethiopia",
"Gabon",
"Gambia",
"Ghana",
"Guinea",
"Guinea-Bissau",
"Kenya",
"Lesotho",
"Liberia",
"Libya",
"Madagascar",
"Malawi",
"Mali",
"Mauritania",
"Mauritius",
"Morocco",
"Mozambique",
"Namibia",
"Niger",
"Nigeria",
"Rwanda",
"Sao Tome and Principe",
"Senegal",
"Seychelles",
"Sierra Leone",
"Somalia",
"South Africa",
"South Sudan",
"Sudan",
"Tanzania",
"Togo",
"Tunisia",
"Uganda",
"Zambia",
"Zimbabwe"]
african_countries_lower = [x.lower() for x in african_countries]
s1['african_vs_non_african'] = s1.country.apply(lambda x: "African" if x.lower() in african_countries_lower else "Non African")
import pylab as pl
s1['african_vs_non_african'].value_counts().plot(kind="bar")
pl.suptitle("African vs Non African (Tendaguru Dinasaurs)")
plt.savefig('../viz_images/AfricanVSnonAfrican_tendaguru.png')
s1['african_vs_non_african'].value_counts()<jupyter_output><empty_output><jupyter_text>Origin of users that post tweets<jupyter_code>s1.columns
s1.to_csv('../data/tweetGeography_tendaguru.csv')
s1['country'].value_counts()[:15].plot(kind="bar")
plt.savefig('UserCountry_tendaguru.png')<jupyter_output><empty_output><jupyter_text>User + Retweets + Replies + Followers (agregation) - Top 20 influencers(diagram), all csv<jupyter_code>tendaguru_clean.columns
tendaguru_clean['followersCount'] = tendaguru_clean.user.apply(lambda x: x['followersCount'])
topn = tendaguru_clean[['username', 'retweetCount', 'replyCount']].groupby('username').sum().sort_values('retweetCount', ascending=False).reset_index()
topn
topInfl = pd.merge(topn, tendaguru_clean[['username', 'followersCount']], how='left', on='username').drop_duplicates().groupby('username').mean().round().reset_index().sort_values('retweetCount', ascending=False)
topInfl
topInfl.to_csv('../data/topInfluencers_tendaguru.csv')
draw_bubble(topInfl, 'followersCount', 'replyCount', 'username', size_ = 'retweetCount',  weight = 60, height=1200, marker_size=None, xaxis= "followers", yaxis = 'retweet')
# fig.write_html("../viz_images/to/topInfluencers_tendaguru.html")<jupyter_output><empty_output><jupyter_text>Google News<jupyter_code>tendaguru_news = pd.read_csv('../data/tendaguru_google_text_manual_clean.csv')
tendaguru_news.shape
tendaguru_news.head()
withDate = tendaguru_news.dropna()
withDate['year'] = withDate.html_date.apply(lambda x:x[:4])
withDate.groupby('year').count()
import pylab as pl
withDate.groupby('year').count()['title'].plot(kind="bar")
pl.suptitle("Google news (Tendaguru Dinasaurs)")
plt.savefig('../viz_images/GoogleNews_tendaguru.png')<jupyter_output><empty_output><jupyter_text>Emotion Bar<jupyter_code>tendaguru_news = ct.clean_tweets(tendaguru_news,content='text',col_name='clean_text',
                general_clean=True,lemma=True,stem=False,remove_tag=True,remove_mention=True,
                remove_emoji=False, remove_stopword=True,min_length=2, untokenized_return=True
                )
import text2emotion as te
tendaguru_news["emotion_dict"] = tendaguru_news['clean_text'].apply(lambda x: te.get_emotion(x))
from collections import Counter
tendaguru_news['emotion_counter'] = tendaguru_news['emotion_dict'].apply(lambda x: Counter(x))
def normalize(d, target=100):
   raw = sum(d.values())
   factor = target/raw
   return {key:value*factor for key,value in d.items()}
gg_emo_dict = normalize(tendaguru_news['emotion_counter'].sum())
plt.bar(gg_emo_dict.keys(), gg_emo_dict.values())
pl.suptitle("News Emotions(Tendaguru Dinasaurs)")
plt.savefig('../viz_images/newsEmotions_tendaguru.png')
tendaguru_news.head()
max(tendaguru_news.iloc[0]['emotion_counter'].values())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15251572327044025, 'fraction_numerical': 0.011455525606469003, 'mean_word_length': 5.761579347000759, 'pattern_counts': {'":': 4, '<': 41, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '431907', 'license': 'Apache-2.0', 'n_tokens_mistral': 3390, 'n_tokens_neox': 3225, 'n_words': 544, 'path': 'notebooks/viz_tendaguru.ipynb', 'repo_name': 'TamatiB/restitution_africa2021', 'size': '8911'}