<jupyter_start><jupyter_text>Coronavirus COVID-19 Tweets Introduction>Corona-virus eda analysis that uses Twitter API, **tweepy** and Python package to explore the tweets generated by users.- toc: true- badges: true- comments: true- categories: [padelpy, corona-virus, scikit-learn]- image: https://www.googleapis.com/download/storage/v1/b/kaggle-user-content/o/inbox%2F769452%2F35db2dd68238bfd958efdabebc9fef8f%2Fcovid-19-4961257_1280-e1586986896105.jpg?generation=1595760042647275&alt=media  Data preparation Load packages<jupyter_code>import numpy as np 
import pandas as pd
import matplotlib
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline 
from wordcloud import WordCloud, STOPWORDS<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>tweets_df = pd.read_csv("covid19_tweets.csv")<jupyter_output><empty_output><jupyter_text>Data exploration Glimpse the data<jupyter_code>print(f"data shape: {tweets_df.shape}")
tweets_df.info()
tweets_df.describe()
tweets_df.head()<jupyter_output><empty_output><jupyter_text>Missing data<jupyter_code>def missing_data(data):
    total = data.isnull().sum()
    percent = (data.isnull().sum()/data.isnull().count()*100)
    tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
    types = []
    for col in data.columns:
        dtype = str(data[col].dtype)
        types.append(dtype)
    tt['Types'] = types
    return(np.transpose(tt))
missing_data(tweets_df)<jupyter_output><empty_output><jupyter_text>Unique values<jupyter_code>def unique_values(data):
    total = data.count()
    tt = pd.DataFrame(total)
    tt.columns = ['Total']
    uniques = []
    for col in data.columns:
        unique = data[col].nunique()
        uniques.append(unique)
    tt['Uniques'] = uniques
    return(np.transpose(tt))
unique_values(tweets_df)<jupyter_output><empty_output><jupyter_text>Most frequent values<jupyter_code>def most_frequent_values(data):
    total = data.count()
    tt = pd.DataFrame(total)
    tt.columns = ['Total']
    items = []
    vals = []
    for col in data.columns:
        itm = data[col].value_counts().index[0]
        val = data[col].value_counts().values[0]
        items.append(itm)
        vals.append(val)
    tt['Most frequent item'] = items
    tt['Frequence'] = vals
    tt['Percent from total'] = np.round(vals / total * 100, 3)
    return(np.transpose(tt))
most_frequent_values(tweets_df)<jupyter_output><empty_output><jupyter_text>Visualize the data distribution<jupyter_code>def plot_count(feature, title, df, size=1, ordered=True):
    f, ax = plt.subplots(1,1, figsize=(4*size,4))
    total = float(len(df))
    if ordered:
        g = sns.countplot(df[feature], order = df[feature].value_counts().index[:20], palette='Set3')
    else:
        g = sns.countplot(df[feature], palette='Set3')
    g.set_title("Number and percentage of {}".format(title))
    if(size > 2):
        plt.xticks(rotation=90, size=8)
    for p in ax.patches:
        height = p.get_height()
        ax.text(p.get_x()+p.get_width()/2.,
                height + 3,
                '{:1.2f}%'.format(100*height/total),
                ha="center") 
    plt.show()<jupyter_output><empty_output><jupyter_text>User name<jupyter_code>plot_count("user_name", "User name", tweets_df,4)<jupyter_output><empty_output><jupyter_text>User location<jupyter_code>plot_count("user_location", "User location", tweets_df,4)<jupyter_output><empty_output><jupyter_text>Tweet source<jupyter_code>plot_count("source", "Source", tweets_df,4)
stopwords = set(STOPWORDS)

def show_wordcloud(data, title = None):
    wordcloud = WordCloud(
        background_color='white',
        stopwords=stopwords,
        max_words=50,
        max_font_size=40, 
        scale=5,
        random_state=1
    ).generate(str(data))

    fig = plt.figure(1, figsize=(10,10))
    plt.axis('off')
    if title: 
        fig.suptitle(title, fontsize=20)
        fig.subplots_adjust(top=2.3)

    plt.imshow(wordcloud)
    plt.show()<jupyter_output><empty_output><jupyter_text>Text wordcloauds<jupyter_code>show_wordcloud(tweets_df['text'], title = 'Prevalent words in tweets')
india_df = tweets_df.loc[tweets_df.user_location=="India"]
show_wordcloud(india_df['text'], title = 'Prevalent words in tweets from India')
us_df = tweets_df.loc[tweets_df.user_location=="United States"]
show_wordcloud(us_df['text'], title = 'Prevalent words in tweets from US')
us_df = tweets_df.loc[tweets_df.user_location=="United Kingdom"]
show_wordcloud(us_df['text'], title = 'Prevalent words in tweets from UK')
us_df = tweets_df.loc[tweets_df.user_location=="Canada"]
show_wordcloud(us_df['text'], title = 'Prevalent words in tweets from Canada')
india_df = tweets_df.loc[tweets_df.user_location=="South Africa"]
show_wordcloud(india_df['text'], title = 'Prevalent words in tweets from South Africa')
india_df = tweets_df.loc[tweets_df.user_location=="Switzerland"]
show_wordcloud(india_df['text'], title = 'Prevalent words in tweets from Switzerland')
us_df = tweets_df.loc[tweets_df.user_location=="London"]
show_wordcloud(us_df['text'], title = 'Prevalent words in tweets from London')<jupyter_output><empty_output><jupyter_text>Hashtags analysis<jupyter_code>def plot_features_distribution(features, title, df, isLog=False):
    plt.figure(figsize=(12,6))
    plt.title(title)
    for feature in features:
        if(isLog):
            sns.distplot(np.log1p(df[feature]),kde=True,hist=False, bins=120, label=feature)
        else:
            sns.distplot(df[feature],kde=True,hist=False, bins=120, label=feature)
    plt.xlabel('')
    plt.legend()
    plt.show()

tweets_df['hashtags'] = tweets_df['hashtags'].replace(np.nan, "['None']", regex=True)
tweets_df['hashtags'] = tweets_df['hashtags'].apply(lambda x: x.replace('\\N',''))
tweets_df['hashtags_count'] = tweets_df['hashtags'].apply(lambda x: len(x.split(',')))
plot_features_distribution(['hashtags_count'], 'Hashtags per tweet (all data)', tweets_df)
tweets_df['hashtags_individual'] = tweets_df['hashtags'].apply(lambda x: x.split(','))
from itertools import chain
all_hashtags = set(chain.from_iterable(list(tweets_df['hashtags_individual'])))
print(f"There are totally: {len(all_hashtags)}")
show_wordcloud(tweets_df['hashtags_individual'], title = 'Prevalent words in hashtags')<jupyter_output><empty_output><jupyter_text>Extract country from locationWe load the country list from the additional database we added to this Notebook. We also create a `country` column in the original dataset.<jupyter_code>country_df = pd.read_csv("../input/iso-country-codes-global/wikipedia-iso-country-codes.csv")
country_df.columns = ["country", "alpha2", "alpha3", "numeric", "iso"]
country_df.head()
tweets_df['country'] = tweets_df['user_location']<jupyter_output><empty_output><jupyter_text>We merge the countries dataset with the tweets dataset.<jupyter_code>tweets_df = tweets_df.merge(country_df, on="country")
tweets_df.head(10)
tw_add_df = tweets_df.groupby(["country", "iso", "alpha3"])['text'].count().reset_index()
tw_add_df.columns = ["country", "iso", "alpha3", "tweets"]
import plotly.express as px

def plot_map(dd_df, title):
    hover_text = []
    for index, row in dd_df.iterrows():
        hover_text.append((f"country: {row['country']}<br>tweets: {row['tweets']}\
                          <br>country code: {row['iso']}<br>country alpha3: {row['alpha3']}"))
    dd_df['hover_text'] = hover_text

    fig = px.choropleth(dd_df, 
                        locations="alpha3",
                        hover_name='hover_text',
                        color="tweets",
                        projection="natural earth",
                        color_continuous_scale=px.colors.sequential.Plasma,
                        width=900, height=700)
    fig.update_geos(   
        showcoastlines=True, coastlinecolor="DarkBlue",
        showland=True, landcolor="LightGrey",
        showocean=True, oceancolor="LightBlue",
        showlakes=True, lakecolor="Blue",
        showrivers=True, rivercolor="Blue",
        showcountries=True, countrycolor="DarkBlue"
    )
    fig.update_layout(title = title, geo_scope="world")
    fig.show()    
plot_map(tw_add_df, "Tweets per country (where country is specified)")<jupyter_output><empty_output><jupyter_text>Extract date and time features<jupyter_code>tweets_df['datedt'] = pd.to_datetime(tweets_df['date'])
tweets_df['year'] = tweets_df['datedt'].dt.year
tweets_df['month'] = tweets_df['datedt'].dt.month
tweets_df['day'] = tweets_df['datedt'].dt.day
tweets_df['dayofweek'] = tweets_df['datedt'].dt.dayofweek
tweets_df['hour'] = tweets_df['datedt'].dt.hour
tweets_df['minute'] = tweets_df['datedt'].dt.minute
tweets_df['dayofyear'] = tweets_df['datedt'].dt.dayofyear
tweets_df['date_only'] = tweets_df['datedt'].dt.date<jupyter_output><empty_output><jupyter_text>Time variation<jupyter_code>tweets_agg_df = tweets_df.groupby(["date_only"])["text"].count().reset_index()
tweets_agg_df.columns = ["date_only", "count"]
def plot_time_variation(df, x='date_only', y='count', hue=None, size=1, title="", is_log=False):
    f, ax = plt.subplots(1,1, figsize=(4*size,3*size))
    g = sns.lineplot(x=x, y=y, hue=hue, data=df)
    plt.xticks(rotation=90)
    if hue:
        plt.title(f'{y} grouped by {hue} | {title}')
    else:
        plt.title(f'{y} | {title}')
    if(is_log):
        ax.set(yscale="log")
    ax.grid(color='black', linestyle='dotted', linewidth=0.75)
    plt.show() 
plot_time_variation(tweets_agg_df, title="Number of tweets / day of year",size=3)
plot_count("dayofweek", "tweets / day of week", tweets_df, size=3, ordered=False)
plot_count("dayofyear", "tweets / day of year", tweets_df, size=3, ordered=False)
plot_count("date_only", "tweets / date", tweets_df,size=4, ordered=False)
plot_count("hour", "tweets / hour", tweets_df,size=4, ordered=False)
plot_count("minute", "tweets / minute", tweets_df,size=5, ordered=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14946117433779837, 'fraction_numerical': 0.015912982173431362, 'mean_word_length': 5.0145366444579045, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 70, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '402357', 'license': 'Apache-2.0', 'n_tokens_mistral': 3719, 'n_tokens_neox': 3557, 'n_words': 588, 'path': '_notebooks/2022-03-18-covid19.ipynb', 'repo_name': 'PrinceExpert17/bettertech', 'size': '9929'}