<jupyter_start><jupyter_text>Hard Classification<jupyter_code># find the index of the topic with largest vector
hard_topic = np.argmax(nmf_topic_vectors,axis=1)

# hard encode the topic
df_hard_topic=pd.get_dummies(hard_topic)

# combine the two dataframes
df_hard_class = pd.concat([df.reset_index(), df_hard_topic],axis=1)

# drop some columns
df_hard_class.drop(['index'],axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Soft Classification<jupyter_code># adding column vecs for the soft classification
df['vecs'] = nmf_topic_vectors.tolist()

# Create a Numpy array of the document vectors
df_np_vecs = np.zeros((len(df),5))
for i, vec in enumerate(df.vecs):
    df_np_vecs[i, :] = vec

# Adding the numpy array (vectors) to the original dataframe
df_full = pd.concat([df.reset_index(), pd.DataFrame(df_np_vecs)], axis=1)

## Drop some columns
df_full.drop(['index', 'vecs'],axis=1, inplace=True)
# pickle the full dataframe
df_full.to_pickle('dataframe_with_vectors.pkl')
df_full.shape<jupyter_output><empty_output><jupyter_text>Daily Analysis - Hard Classification<jupyter_code># change "created_at" to a datetime format
df_hard_class_dt = df_hard_class.copy()
df_hard_class_dt['time']=pd.to_datetime(df["created_at"])
df_hard_class_dt.drop(columns = 'created_at', inplace=True)
# pickle the hard classification
df_hard_class_dt.to_pickle('hard_class.pkl')
with open('hard_class.pkl','rb') as file:
    df_hard_class_dt = pickle.load(file)
# remove data generated on the 18th and the 22nd by creating a mask
mask = (df_hard_class_dt['time'].dt.day==22) | (df_hard_class_dt['time'].dt.day==18)

# create a new dataframe
df_hard_class_days = df_hard_class_dt[~mask]
df_hard_class_days.tail(5)
def day_byhour(day,starthour, duration=3, df=df_hard_class_days):
    df_hour = df.loc[((df['time'].dt.day==day) & (starthour<=df['time'].dt.hour) 
                                  &(df['time'].dt.hour<(starthour+duration))), 
                                 [0,1,2,3,4]].sum(axis=0)
    return pd.DataFrame(df_hour, columns=['{}-{}'.format(day,starthour)])
def hourblock(df, duration):
    all_df = []
    for day in [19,20,21]:
        for hour in range(0,24,duration):
            all_df.append(day_byhour(day, hour, duration, df))
    hourly_topic = pd.concat(all_df, axis=1)
    return hourly_topic
df_daybyhour_analysis = hourblock(df_hard_class_days, 3)
df_daybyhour_analysis
# rename the column names so graphs will show in chronological order
df_daybyhour_analysis.rename(columns = {'19-3':'19-03', '19-6':'19-06','19-9':'19-09',
                                        '20-3':'20-03', '20-6':'20-06','20-9':'20-09',
                                        '21-3':'21-03', '21-6':'21-06','21-9':'21-09'},inplace=True)
plt.figure(figsize=(22,15))
plt.title('Number of Tweets by Topic - Hourly', fontsize=35)
plt.plot(df_daybyhour_analysis.columns,df_daybyhour_analysis.iloc[0], label='Wedding & Queen')
plt.plot(df_daybyhour_analysis.columns,df_daybyhour_analysis.iloc[1], label='Wedding Guests')
plt.plot(df_daybyhour_analysis.columns,df_daybyhour_analysis.iloc[2], label='Congrats Meghan & Harry')
plt.plot(df_daybyhour_analysis.columns,df_daybyhour_analysis.iloc[3], label='The Royal Family')
plt.plot(df_daybyhour_analysis.columns,df_daybyhour_analysis.iloc[4], label="Bishop Curry's Sermon & Love")
plt.legend(loc='upper right',fontsize=30)
plt.xticks(fontsize=20, rotation=60)
plt.yticks(fontsize=20)
plt.xlabel('Day and Hour',fontsize=30)
plt.ylabel('Number of Tweets',fontsize=30)
plt.savefig('number of tweets by topic - hourly.png', bbox_inches='tight');<jupyter_output><empty_output><jupyter_text>Daily Analysis - Soft Prediction<jupyter_code>with open('dataframe_with_vectors','rb') as file:
    full_df = pickle.load(file)
full_df.head(5)
# change "created_at" to a datetime format
df_soft_class_dt = full_df.copy()
df_soft_class_dt['time']=pd.to_datetime(full_df["created_at"])
df_soft_class_dt.drop(columns = 'created_at', inplace=True)
# remove data generated on the 18th and the 22nd by creating a mask
mask = (df_soft_class_dt['time'].dt.day==22) | (df_soft_class_dt['time'].dt.day==18)

full_df_days = df_soft_class_dt[~mask]
def day_byhourper(day,starthour, duration=3, df=full_df_days):
    df_hour = df.loc[((df['time'].dt.day==day) & (starthour<=df['time'].dt.hour) 
                                  &(df['time'].dt.hour<(starthour+duration))), 
                                 [0,1,2,3,4]].mean(axis=0)
    return pd.DataFrame(df_hour, columns=['{}-{}'.format(day,starthour)])
def hourblockper(df, duration):
    all_dfper = []
    for day in [19,20,21]:
        for hour in range(0,24,duration):
            all_dfper.append(day_byhourper(day, hour, duration, df))
    hourly_topicper = pd.concat(all_dfper, axis=1)
    return hourly_topicper
df_daybyhourper_analysis = hourblockper(full_df_days, 3)
df_daybyhourper_analysis<jupyter_output><empty_output><jupyter_text>KMeans to find the cluster mean<jupyter_code>from sklearn.cluster import KMeans
km = KMeans(n_clusters=4)
clusts = km.fit_predict(nmf_topic_vectors)
cluster_vecs = km.cluster_centers_
cluster_vecs
np.unique(clusts, return_counts=True)
clusts.shape
clusts[:10]<jupyter_output><empty_output><jupyter_text>KMeans: find number of clusters using inertia<jupyter_code># compute inertia (SSE) over multiple values of k and try to find the "elbow" point
SSE = []
for k in range(2,20):
    km = KMeans(n_clusters=k, random_state=42)
    km.fit(nmf_topic_vectors)
    SSE.append(km.inertia_)
# plot the inertia and visualize the elbow point
plt.figure(figsize=(20,15))
plt.title('Inertia by Number of Clusters',fontsize=35)
k_clusters = range(2,20)
plt.plot(k_clusters, SSE)
plt.xlabel('Number of Clusters',fontsize=25)
plt.ylabel('Inertia / SSE', fontsize=25)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
plt.xticks(np.arange(2, 21, 2.0))
plt.savefig('Inertia by Number of Clusters.png', bbox_inches='tight');
# try cluster = 6, seems like the most reasonable number of clusters
estimator = KMeans(n_clusters=6)
clusts = estimator.fit_predict(nmf_topic_vectors)
# find the vectors for these six clusters
cluster_vecs = estimator.cluster_centers_
cluster_vecs
# find the number of tweets (documents) that belong to each of the cluster
np.unique(clusts, return_counts=True)
df_cluster = pd.DataFrame(clusts, columns=['cluster'])
df_full_cluster = df_hard_class_dt.join(df_cluster)
df_full_cluster<jupyter_output><empty_output><jupyter_text>tSNE<jupyter_code>from sklearn.manifold import TSNE
tsne = TSNE(n_components=2, verbose=1, perplexity=200, n_iter=300)
X_ne = tsne.fit_transform(nmf_topic_vectors[400000:406000])
plt.figure(figsize=(20, 15))
plt.title('Clustering using tSNE, k=6 (only 6000 datapoints)',fontsize=30)
plt.scatter(X_ne[:, 0], X_ne[:, 1],alpha=0.15,c=clusts[400000:406000],cmap=plt.cm.rainbow)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
plt.savefig('tSNE perplexity=200.png', bbox_inches='tight');<jupyter_output><empty_output><jupyter_text>Find my tweet<jupyter_code>df_full_cluster[df_full_cluster['screen_name']=='miranda_zhao']

## mostly about the wedding, tiny bit of Royal Wedding
df_full_cluster[df_full_cluster['screen_name']=='miranda_zhao']['cleaned_text'].values
df_full_cluster[df_full_cluster['screen_name']=='bbcworldservice']

## mostly about wedding and the queen, and a little bit of Congratulations Meghan and Harry<jupyter_output><empty_output><jupyter_text>Location Analysis on the Tweets<jupyter_code># Refer to Step 3_Location on the pickled dataframe
with open('locationdataframe.pkl','rb') as file:
    locationdf = pickle.load(file)
df_location = pd.merge(df_full_cluster, locationdf)
df_location.drop('created_at',axis=1,inplace=True)
df_location.head(5)
df_location.country.value_counts()
df_location.to_pickle('location_analysis_df.pkl')
# For the US, define topics 
df_usa = df_location.loc[(df_location['country']=='usa'),['cluster','cleaned_text']]
df_usa.cluster.value_counts(normalize=True)

#wedding
# For the UK, define topics
df_uk = df_location.loc[(df_location['country']=='uk'),['cluster','cleaned_text']]
df_uk.cluster.value_counts(normalize=True)

#wedding 
# For the rest of the world, define topics
df_other = df_location.loc[(df_location['country']=='other'),['cluster','cleaned_text']]
df_other.cluster.value_counts(normalize=True)

# cluster 0 --> Royal Family & Congratulations Meghan & Harry
#Who talked about the sermon?
df_vector5 = df_location.loc[(df_location['cluster']==5),['country']]
df_vector5.country.value_counts()<jupyter_output><empty_output><jupyter_text>Sentiment Analysis on tweets based on location<jupyter_code>from textblob import TextBlob
# 0 is very objective, 1 is very subjective
TextBlob("i dislike datascience.").sentiment
polarity_usa = []
subjectivity_usa=[]
for i in df_usa['cleaned_text']:
    polarity_usa.append(TextBlob(i).sentiment[0])
    subjectivity_usa.append(TextBlob(i).sentiment[1])

print(np.mean(polarity_usa))
print(np.mean(subjectivity_usa))
polarity_uk = []
subjectivity_uk=[]
for i in df_uk['cleaned_text']:
    polarity_uk.append(TextBlob(i).sentiment[0])
    subjectivity_uk.append(TextBlob(i).sentiment[1])

print(np.mean(polarity_uk))
print(np.mean(subjectivity_uk))
polarity_other = []
subjectivity_other=[]
for i in df_other['cleaned_text']:
    polarity_other.append(TextBlob(i).sentiment[0])
    subjectivity_other.append(TextBlob(i).sentiment[1])

print(np.mean(polarity_other))
print(np.mean(subjectivity_other))<jupyter_output>0.10557816002783182
0.24237132212871232
{'fraction_non_alphanumeric': 0.138108761964868, 'fraction_numerical': 0.029872725360260862, 'mean_word_length': 6.445575567736883, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '564114', 'license': 'MIT', 'n_tokens_mistral': 3707, 'n_tokens_neox': 3466, 'n_words': 614, 'path': 'Code/Step4_ClusteringandSegmentation.ipynb', 'repo_name': 'mizhao2018/TwitterAnalysisonRoyalWedding', 'size': '9507'}