<jupyter_start><jupyter_text>Combined 1,2,3 grams<jupyter_code># 

vectorizer = CountVectorizer(max_features=5000, # only top 5k words
                             min_df=2,          # words must appear in this many speeches to count
                             #max_df=0.9,        # words can't appear in more % of speeches than this.  
                             stop_words=sw,#stopwords.words('english'),
                             tokenizer=my_tokenizer,
                             ngram_range=(3,3)  # (1,3) combined unigrams, bigrams, trigrams
                            )

counts = vectorizer.fit_transform(bow['speech'])
counts_df = pd.DataFrame(counts.toarray())
counts_df.index.name = 'speech'
#tfidf.columns = tfidf.apply(lambda x: vocab_r[x.name])
counts_df.columns = vectorizer.get_feature_names()
counts_df.head(3)
# convert counts into tfidf
transformer = TfidfTransformer(smooth_idf=False)
tfidf = transformer.fit_transform(counts)
tfidf_df = pd.DataFrame(tfidf.toarray())
tfidf_df.index.name = 'speech'
tfidf_df.columns = vectorizer.get_feature_names()
tfidf_df['videoId'] = bow.index.values
tfidf_df = tfidf_df.set_index('videoId')
tfidf_df['_speaker'] = tfidf_df.apply(lambda x: bow.loc[x.name].speaker, 1)
tfidf_df = tfidf_df.reset_index().set_index(['videoId','_speaker'])
tfidf_df.head()
# collapse to speaker-only
tfidf_speaker = tfidf_df.reset_index().drop(columns="videoId").groupby(['_speaker']).mean()
# Normalize doc vector lengths
tfidf_speaker = tfidf_speaker.apply(lambda x: x / norm(x), 1)
# transpose for easier filtering
tfidf_speaker = tfidf_speaker.T.reset_index()
tfidf_speaker.sort_values('biden', ascending=False).head(10)#[['index']].values
# alternate method of speaker-only TFIDF
# collapse speech-level counts first
counts_df['videoId'] = bow.index.values
counts_df = counts_df.set_index('videoId')
counts_df['_speaker'] = counts_df.apply(lambda x: bow.loc[x.name].speaker, 1)
counts_df = counts_df.reset_index().set_index(['_speaker']).drop(columns=['videoId'])

# convert counts into tfidf
# use actual term frequency?
#counts_speaker = counts_df.sum(level='_speaker').div(counts_df.sum(level='_speaker').sum(axis=1), axis='rows')
transformer = TfidfTransformer(use_idf=True,smooth_idf=True)
tfidf_speaker2 = transformer.fit_transform(counts_df.sum(level='_speaker'))
tfidf_speaker2 = pd.DataFrame(tfidf_speaker2.toarray())
tfidf_speaker2.columns = vectorizer.get_feature_names()
tfidf_speaker2['_speaker'] = counts_df.sum(level='_speaker').index.values
tfidf_speaker2 = tfidf_speaker2.set_index('_speaker')
tfidf_speaker2.index.name = 'speaker'
# Normalize doc vector lengths
#tfidf_speaker2 = tfidf_speaker2.apply(lambda x: x / norm(x), 1)
# transpose for easier filtering
tfidf_speaker2 = tfidf_speaker2.T.reset_index()
top10 = pd.DataFrame(tfidf_speaker2.sort_values('biden', ascending=False).head(10)[['index']].values)
top10.columns = ['biden']
top10['trump'] = tfidf_speaker2.sort_values('trump', ascending=False).head(10)[['index']].values
top10['harris'] = tfidf_speaker2.sort_values('harris', ascending=False).head(10)[['index']].values
top10['pence'] = tfidf_speaker2.sort_values('pence', ascending=False).head(10)[['index']].values
top10
top10 = pd.DataFrame(tfidf_speaker.sort_values('biden', ascending=False).head(10)[['index']].values)
top10.columns = ['biden']
top10['trump'] = tfidf_speaker.sort_values('trump', ascending=False).head(10)[['index']].values
top10['harris'] = tfidf_speaker.sort_values('harris', ascending=False).head(10)[['index']].values
top10['pence'] = tfidf_speaker.sort_values('pence', ascending=False).head(10)[['index']].values
top10
# compare to speech-collapsed-into-speaker
tfidf_speaker.sort_values('biden', ascending=False).head(10)#[['index']].values
top10 = pd.DataFrame(tfidf_speaker.sort_values('biden', ascending=False).head(10)[['index']].values)
top10.columns = ['biden']
top10['trump'] = tfidf_speaker.sort_values('trump', ascending=False).head(10)[['index']].values
top10['harris'] = tfidf_speaker.sort_values('harris', ascending=False).head(10)[['index']].values
top10['pence'] = tfidf_speaker.sort_values('pence', ascending=False).head(10)[['index']].values
top10
tfidf_speaker['avg'] = np.mean(tfidf_speaker, axis=1)
tfidf_speaker.columns
from sklearn.metrics.pairwise import cosine_similarity
cs = cosine_similarity(tfidf_short)
csdf = pd.DataFrame(data=cs, index=['biden','harris','pence','trump'], columns=['biden','harris','pence','trump'])
csdf
# let's look at cosine similarity for all the speeches
from sklearn.metrics.pairwise import cosine_similarity
tfidf_df_mean = tfidf_df.mean()
tfidf_a = tfidf_df.T
tfidf_a['avg'] = tfidf_df_mean
all_sp = tfidf_a.sort_values(['avg'], ascending=False)\
    .drop(columns=['avg'])\
    .head(30000).T

csa = cosine_similarity(all_sp)
csadf = pd.DataFrame(data=csa, index=bow.index, columns=bow.index)
csadf
csadf.to_csv('speech_similarity.csv')<jupyter_output><empty_output><jupyter_text>Hierarchical Cluster Analysis and Dendrograms<jupyter_code>import scipy.cluster.hierarchy as sch
def hca(sims, linkage_method='ward', color_thresh=.3, figsize=(10, 30)):
    tree = sch.linkage(sims, method=linkage_method)
    labels = bow.speaker.values 
    plt.figure()
    fig, axes = plt.subplots(figsize=figsize)
    dendrogram = sch.dendrogram(tree, 
                                labels=labels, 
                                orientation="left", 
                                count_sort=True,
                                distance_sort=True,
                                above_threshold_color='.75',
                                color_threshold=color_thresh
                               )
    plt.tick_params(axis='both', which='major', labelsize=14)
from scipy.spatial.distance import pdist
from matplotlib import pyplot as plt

all_sp = tfidf_df
L1 = all_sp.apply(lambda x: x / x.sum(), 1)

PAIRS = pd.DataFrame(index=pd.MultiIndex.from_product([bow.index.tolist(), bow.index.tolist()])).reset_index()
PAIRS = PAIRS[PAIRS.level_0 < PAIRS.level_1].set_index(['level_0','level_1'])
PAIRS.index.names = ['speech_a', 'speech_b']

PAIRS['cosine'] = pdist(L1, 'cosine')
PAIRS['cityblock'] = pdist(L1, 'cityblock')

hca(PAIRS['cosine'], color_thresh=2)
#hca(PAIRS['cityblock'], color_thresh=4.6)<jupyter_output><empty_output><jupyter_text>Group 0 (red): Trump rallies1 speech labeled "harris", actually titled "Trump blasts <NAME>, mocks <NAME> at.." bxuykoOlwjA1 "pence" speech, but the first 1.5 hours are random crowd talking, followed by a short Pence speech Group 1 (purple): Formal, "presidential" speeches33/61 speeches Biden15 trump, 11 harris, 2 penceInterestingly, these Trump speeches are not rallies, but major public appearances - SOTU, RNC, 4th of July, 60 minutes, etc. Group 2 (blue): Pence speeches (21/26)1 "biden" actually Pence Group 3 (green): Casual talk7 interviews, a couple press conferences, several miscategorized talk shows<jupyter_code>tree = sch.linkage(PAIRS['cosine'], method='ward')
cuts = sch.cut_tree(tree, height=2)
bow['group'] = cuts
bow['date'] = pd.to_datetime(bow['date'])
ax = bow.sort_values(['date']).reset_index().plot.scatter('date','group')
plt.setp(ax.get_xticklabels(), rotation=45)
plt.title('Clusters plotted by date')
plt.show()
bow[bow['group']==3][['speaker','date']].groupby('speaker').count()
bow[bow['group']==3].sort_values(['speaker'])[['speaker','date','title','transcript_type']].head(30)<jupyter_output><empty_output><jupyter_text>Principal Components Analysis<jupyter_code>from sklearn.decomposition import PCA
from scipy.linalg import norm

# normalize vectors
tfidf_df = tfidf_df.apply(lambda x: x / norm(x), 1)
# center vectors
tfidf_df = tfidf_df - tfidf_df.mean()

pca = PCA(n_components=4)
pca.fit(tfidf_df)

# term loadings = eigenvectors * eigenvalues
loadings = pd.DataFrame(pca.components_).T * np.sqrt(pca.explained_variance_)

# doc-component matrix
dcm = pd.DataFrame(pca.transform(tfidf_df))
dcm.index = [tfidf_df.index[i][0] for i in dcm.index.values]
dcm.index.name = "id"
dcm['title'] = dcm.apply(lambda x: bow.loc[x.name].title, 1)
dcm['speaker'] = dcm.apply(lambda x: bow.loc[x.name].speaker, 1)
dcm['group'] = dcm.apply(lambda x: bow.loc[x.name].group, 1)
dcm['date'] = dcm.apply(lambda x: bow.loc[x.name].date, 1)
import plotly_express as px
fig = px.scatter(dcm, 0, 1, 
                    color='speaker', 
                    hover_name='title')
fig.show()
px.line(dcm[dcm['speaker']=='pence'].reset_index(), x='date', y=[0,1], hover_name='title')
loadings['trump_rally'] = (loadings[0]+loadings[1])*-1
loadings['pence'] = loadings[0]+(loadings[1]*-1)
loadings['interviews'] = loadings[1]+(loadings[0]*-1)
loadings['formal'] = loadings[0]+loadings[1]

# put term strings back as index
loadings.index = [tfidf_df.T.index[i] for i in loadings.index.values]
loadings.index.name = 'term'
loadings['trump_rally'].sort_values(ascending=False).head(10)
loadings['pence'].sort_values(ascending=False).head(10)
loadings[1].sort_values(ascending=False).head(10)
import plotly_express as px
fig = px.scatter(dcm, 0, 1, 
                    color='group', 
                    hover_name='title')
fig.show()<jupyter_output><empty_output><jupyter_text>Collocation - uses PMI (pointwise mutual information) to figure out  whether a bigram/trigram is better treated as a single "word"<jupyter_code># start with unigrams - just p(x)
pmi = counts_df.loc[:,[c for c in counts_df.columns.values if ' ' not in c]]
pmi = pmi.div(pmi.sum(axis=1), axis=0)

# bigrams - log(p(x,y) / (p(x)*p(y)))
pmi2 = counts_df.loc[:,[c for c in counts_df.columns.values if c.count(' ')==1 ]]
#pmi = pmi.div(pmi.sum(axis=1), axis=0)
def pxpy(xy_term):
    x, y = xy_term.name.split()
    #px = pmi[x] + 1e-60
    #py = pmi[y] + 1e-60
    return (pmi[x] + 1e-60) * (pmi[y] + 1e-60) # add a small value to avoid divide by zero

pmi2 = np.log2(pmi2.div(pmi2.apply(pxpy, axis=0), axis=0) + 1e-60)
pmi2.index.name = 'speech'
pmi2['videoId'] = bow.index.values
pmi2 = pmi2.set_index('videoId')
pmi2['_speaker'] = pmi2.apply(lambda x: bow.loc[x.name].speaker, 1)
pmi2 = pmi2.reset_index().set_index(['videoId','_speaker'])
pmi2s = pmi2.reset_index().drop(columns="videoId").groupby(['_speaker']).mean()
pmi2s = pmi2s.T.reset_index()
pmi2s.sort_values('biden', ascending=False).head(10)[['index']].values
top10 = pd.DataFrame(pmi2s.sort_values('biden', ascending=False).head(10)[['index']].values)
top10.columns = ['biden']
top10['trump'] = pmi2s.sort_values('trump', ascending=False).head(10)[['index']].values
top10['harris'] = pmi2s.sort_values('harris', ascending=False).head(10)[['index']].values
top10['pence'] = pmi2s.sort_values('pence', ascending=False).head(10)[['index']].values
top10<jupyter_output><empty_output><jupyter_text>Only bigrams<jupyter_code># create document-term count matrix
vectorizer = CountVectorizer(#max_features=5000, # only top 5k words
                             min_df=3,          # words must appear in this many speeches to count
                             max_df=0.9,        # words can't appear in more % of speeches than this.  
                             stop_words=stopwords.words('english'),
                             tokenizer=my_tokenizer,
                             ngram_range=(2,2)  # only bigrams
                            )
counts = vectorizer.fit_transform(bow['speech'])
counts_df = pd.DataFrame(counts.toarray())
counts_df.index.name = 'speech'
counts_df.columns = vectorizer.get_feature_names()

# convert counts into tfidf
transformer = TfidfTransformer(smooth_idf=False)
tfidf = transformer.fit_transform(counts)
# make it a nice df
tfidf_df = pd.DataFrame(tfidf.toarray())
tfidf_df.index.name = 'speech'
tfidf_df.columns = vectorizer.get_feature_names()
tfidf_df['videoId'] = bow.index.values
tfidf_df = tfidf_df.set_index('videoId')

# group by speaker and aggregate
tfidf_df['_speaker'] = tfidf_df.apply(lambda x: bow.loc[x.name].speaker, 1)
tfidf_df = tfidf_df.reset_index().set_index(['videoId','_speaker'])
# collapse to speaker-only
tfidf_speaker = tfidf_df.reset_index().drop(columns="videoId").groupby(['_speaker']).mean()
# Normalize doc vector lengths
tfidf_speaker = tfidf_speaker.apply(lambda x: x / norm(x), 1)
# transpose for easier filtering
tfidf_speaker = tfidf_speaker.T.reset_index()

# convert df to a handy top-10 comparison matrix
top10 = pd.DataFrame(tfidf_speaker.sort_values('biden', ascending=False).head(10)[['index']].values)
top10.columns = ['biden']
top10['trump'] = tfidf_speaker.sort_values('trump', ascending=False).head(10)[['index']].values
top10['harris'] = tfidf_speaker.sort_values('harris', ascending=False).head(10)[['index']].values
top10['pence'] = tfidf_speaker.sort_values('pence', ascending=False).head(10)[['index']].values
top10
# key phrases - a score for which terms each speaker uses that the others don't.
tfidf_speaker['biden2'] = tfidf_speaker.biden - (tfidf_speaker.harris + tfidf_speaker.pence + tfidf_speaker.trump)
tfidf_speaker['trump2'] = tfidf_speaker.trump - (tfidf_speaker.harris + tfidf_speaker.pence + tfidf_speaker.biden)
tfidf_speaker['harris2'] = tfidf_speaker.harris - (tfidf_speaker.trump + tfidf_speaker.pence + tfidf_speaker.biden)
tfidf_speaker['pence2'] = tfidf_speaker.pence - (tfidf_speaker.harris + tfidf_speaker.trump + tfidf_speaker.biden)

top10 = pd.DataFrame(tfidf_speaker.sort_values('biden2', ascending=False).head(10)[['index']].values)
top10.columns = ['biden']
top10['trump'] = tfidf_speaker.sort_values('trump2', ascending=False).head(10)[['index']].values
top10['harris'] = tfidf_speaker.sort_values('harris2', ascending=False).head(10)[['index']].values
top10['pence'] = tfidf_speaker.sort_values('pence2', ascending=False).head(10)[['index']].values
top10<jupyter_output><empty_output><jupyter_text>Only trigrams<jupyter_code># create document-term count matrix
vectorizer = CountVectorizer(#max_features=5000, # only top 5k words
                             min_df=3,          # words must appear in this many speeches to count
                             max_df=0.9,        # words can't appear in more % of speeches than this.  
                             stop_words=stopwords.words('english'),
                             tokenizer=my_tokenizer,
                             ngram_range=(3,3)  # only bigrams
                            )
counts = vectorizer.fit_transform(bow['speech'])
counts_df = pd.DataFrame(counts.toarray())
counts_df.index.name = 'speech'
counts_df.columns = vectorizer.get_feature_names()

# convert counts into tfidf
transformer = TfidfTransformer(smooth_idf=False)
tfidf = transformer.fit_transform(counts)
# make it a nice df
tfidf_df = pd.DataFrame(tfidf.toarray())
tfidf_df.index.name = 'speech'
tfidf_df.columns = vectorizer.get_feature_names()
tfidf_df['videoId'] = bow.index.values
tfidf_df = tfidf_df.set_index('videoId')

# group by speaker and aggregate
tfidf_df['_speaker'] = tfidf_df.apply(lambda x: bow.loc[x.name].speaker, 1)
tfidf_df = tfidf_df.reset_index().set_index(['videoId','_speaker'])
# collapse to speaker-only
tfidf_speaker = tfidf_df.reset_index().drop(columns="videoId").groupby(['_speaker']).mean()
# Normalize doc vector lengths
tfidf_speaker = tfidf_speaker.apply(lambda x: x / norm(x), 1)
# transpose for easier filtering
tfidf_speaker = tfidf_speaker.T.reset_index()

# convert df to a handy top-10 comparison matrix
top10 = pd.DataFrame(tfidf_speaker.sort_values('biden', ascending=False).head(10)[['index']].values)
top10.columns = ['biden']
top10['trump'] = tfidf_speaker.sort_values('trump', ascending=False).head(10)[['index']].values
top10['harris'] = tfidf_speaker.sort_values('harris', ascending=False).head(10)[['index']].values
top10['pence'] = tfidf_speaker.sort_values('pence', ascending=False).head(10)[['index']].values
top10
tfidf_speaker
# key phrases - a score for which terms each speaker uses that the others don't.
tfidf_speaker['biden2'] = tfidf_speaker.biden - (tfidf_speaker.harris + tfidf_speaker.pence + tfidf_speaker.trump)
tfidf_speaker['trump2'] = tfidf_speaker.trump - (tfidf_speaker.harris + tfidf_speaker.pence + tfidf_speaker.biden)
tfidf_speaker['harris2'] = tfidf_speaker.harris - (tfidf_speaker.trump + tfidf_speaker.pence + tfidf_speaker.biden)
tfidf_speaker['pence2'] = tfidf_speaker.pence - (tfidf_speaker.harris + tfidf_speaker.trump + tfidf_speaker.biden)

top10 = pd.DataFrame(tfidf_speaker.sort_values('biden2', ascending=False).head(10)[['index']].values)
top10.columns = ['biden']
top10['trump'] = tfidf_speaker.sort_values('trump2', ascending=False).head(10)[['index']].values
top10['harris'] = tfidf_speaker.sort_values('harris2', ascending=False).head(10)[['index']].values
top10['pence'] = tfidf_speaker.sort_values('pence2', ascending=False).head(10)[['index']].values
top10
tfidf_df = tfidf_df.reset_index().set_index('videoId')
tfidf_df['vdate'] = tfidf_df.apply(lambda x: bow.loc[x.name].date, 1)
tfidf_df
px.scatter(tfidf_df, "vdate","know", color="_speaker")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15490219184539242, 'fraction_numerical': 0.022566580249823238, 'mean_word_length': 5.160798548094374, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '474242', 'license': 'MIT', 'n_tokens_mistral': 6587, 'n_tokens_neox': 6228, 'n_words': 1020, 'path': 'top_tfidf_with_ngrams.ipynb', 'repo_name': 'kfinity/capstone-speeches', 'size': '16982'}