<jupyter_start><jupyter_text>For Gephi<jupyter_code>data.head()
# Find the top 30 words said in each speech
top_dict = {}
for c in data.columns:
    top = data[c].sort_values(ascending=False).head(30)
    top_dict[c]= list(zip(top.index, top.values))
words = []
for speech in data.columns:
    top = [word for (word, count) in top_dict[speech]]
    for t in top:
        words.append(t)
        
boring_words = ['say','like','just','dont','don','im',
                'ive','youll','youve','things','thing','youre','right','really','lot',
                'make','know','people','way',
                'come','thats','graduate']
len(words)
most_common = pd.DataFrame(Counter([word for word in words if word not in boring_words]).most_common()[1:6000:20],
                          columns=['word','freq'])
len(most_common),most_common
dtm.head()
columns_drop = [word for word in data.index.tolist() if word not in most_common.word.tolist()]
len(columns_drop)
data.head()
data_slim = dtm.drop(columns=columns_drop)
data_slim
# Create co-currence matrix
df = data_slim.transpose().dot(data_slim)
df.values[[np.arange(df.shape[0])]*2] = 0
df
m = min(min(df.to_numpy().tolist()))
df = df/10
df
df.to_csv('../dump/to_gephi.csv',sep=',')
# Create co-currence matrix
df_fat = data.transpose().dot(data)
df_fat.to_csv('../dump/to_gephi_fat.csv',sep=',')<jupyter_output><empty_output><jupyter_text>Topic & terms for Gephi<jupyter_code>topic_df
topic_df['words'] = topic_df.apply(lambda x: " ".join(x.astype('str')),axis=1)
topic_word = topic_df[['words']]
topic_word
data_dtm = cv_dtm(topic_word,'words')
data_dtm
# Create topic-topic df to append
topic_topic = pd.DataFrame(columns = data_dtm.index,index=data_dtm.index).fillna(0)
topic_topic
concat_1 = pd.concat([topic_topic,data_dtm],axis=1)
concat_1
# Create word-word df to append
word_word = pd.DataFrame(columns = concat_1.columns,index=data_dtm.columns).fillna(0)
word_word
concat_2 = pd.concat([concat_1,word_word],axis=0)
concat_2
concat_2.to_csv('../dump/to_gephi_topic_words.csv',sep=',')
concat_2.to_pickle('../dump/to_nx_topic_words')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17186024551463644, 'fraction_numerical': 0.013692162417374882, 'mean_word_length': 5.269230769230769, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '490767', 'license': 'MIT', 'n_tokens_mistral': 813, 'n_tokens_neox': 793, 'n_words': 133, 'path': 'notebook/network_3.ipynb', 'repo_name': 'katiehuang1221/onl_ds5_project_4', 'size': '2118'}