<jupyter_start><jupyter_text>Analyze state of the union addresses. Data source: https://en.wikisource.org/wiki/Portal:State_of_the_Union_Speeches_by_United_States_PresidentsFind similar or dissimilar speeches using word embeddings https://nlp.stanford.edu/IR-book/html/htmledition/sublinear-tf-scaling-1.html https://www.oreilly.com/library/view/applied-text-analysis/9781491963036/ch04.html   Setup<jupyter_code>from bs4 import BeautifulSoup
import requests
import string
import pandas as pd
import urllib.request
import re
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import spacy
import time
from sklearn.neighbors import NearestNeighbors
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import AgglomerativeClustering
import scipy.cluster.hierarchy as sch

# Data source we are going to scrape for results
data_url = 'https://en.wikisource.org/wiki/Portal:State_of_the_Union_Speeches_by_United_States_Presidents'

link_list = []

# extract the text of a speech from a URL
# text is extracted in a list of paragraphs (strings) for each speech
def get_speech(url):
    return([ p.text.strip() for p in BeautifulSoup(urllib.request.urlopen(url)).find_all("p") if \
             'This work is in the public domain in the United States because it is a work of the United States federal government' \
             not in p.text.strip()])

# Make a frequency count by distinct values of 
# column(s) listed in 'groupbyvars'
# Returns pandas dataframe
def tidy_count(df,groupbyvars):
    return(df.groupby(groupbyvars).size().reset_index().\
        rename(columns={0: "n"}).sort_values('n',ascending=False).reset_index(drop=True))<jupyter_output><empty_output><jupyter_text>Web Scraping<jupyter_code>resp = urllib.request.urlopen(data_url)
soup = BeautifulSoup(resp, from_encoding=resp.info().get_param('charset'))

# Get all links to state of the union addresses from 
for link in soup.find_all('a', href=True):
    if "union_address" in link['href'].lower() or "union_speech" in link['href'].lower() \
        and "portal" not in link['href'].lower() and "#" not in link['href'].lower():
        link_list.append(link['href'])

# Note that I am storing these speeches as lists of paragraphs (strings) for readability
speeches = [get_speech('https://en.wikisource.org' + link) for link in link_list]
# Extract presidents names from link text
presidents = [ link.replace('%','/').split('/')[2].replace('_',' ') for link in link_list ]

# Extract state of the union text entries so we can extract the date
sou_entries = []
for item in soup.find_all('li'):
    if 'union' in item.text.strip().lower() and '(' in  item.text.strip().lower():
        sou_entries.append(item.text.strip())

speeches_pd = pd.DataFrame({
                'president' : presidents,
                'speech' : speeches,
                'year' : [int(re.findall('\d+',item)[1]) for item in sou_entries ]} )
len(speeches_pd)
#speeches_pd['speech_num'] = speeches_pd.index # for joining
speeches_pd.sample(n=5,random_state=42)<jupyter_output><empty_output><jupyter_text>Preprocessing Clean text (remove stop words, convert to lower case, remove non-alphabetic content)<jupyter_code>nlp = spacy.load('en_core_web_lg', disable=['parser', 'ner']) # disabling the parser makes it run faster
#nlp = spacy.load('en_core_web_lg',disable=['parser', 'ner'])

## Workaround for stopwords bug in en_core_web_lg model
## only need if you are using the .is_stop spacy attribute
for word in nlp.Defaults.stop_words:
     lex = nlp.vocab[word]
     lex.is_stop = True

# Cleans a string by tokenizing with spacy, removing trailing and leading non-alphabetic characters,
# and then removing stop words and tokens that have no alphabetic characters

def spacy_preprocess(text): 
    text_out = []
    # Only keep alphabetic characeters, spaces, and some select characters
    for token in nlp(text.lower()):
        # remove traililng or leading non-alphabetic text
        cleaned_token = token.text.strip(string.punctuation + string.digits + string.whitespace)

        # remove stop words and tokens with no alphabetic content
        if len(cleaned_token) > 1 and cleaned_token not in nlp.Defaults.stop_words and \
        any(c.isalpha() for c in cleaned_token):
            text_out.append(cleaned_token)
    return(" ".join(text_out))

# Deprecated - Preprocess pipeline with spacy. 
# def spacy_preprocess(text): 
#     text_out = []
#     for token in nlp(text.lower()):
#         # Get rid of stop words and non-alphanumeric
#         if not token.is_stop and token.is_alpha and len(token)>1:
#                 text_out.append(token.text)            
#     return(nlp(" ".join(text_out)))
# Print stop words
#print(nlp.Defaults.stop_words)
for token in nlp('The the weather'):
    print(token.is_stop)
# test spacy preprocessing
spacy_preprocess('The dog ran into Bob beCause he saw 234 squirrels under VAU15')
test_sentence = """d3.js  Programming ### experience$ /battleship# I\ 
    * also program #and in -python- - sometimes## te\rminal// """

print(spacy_preprocess(test_sentence))<jupyter_output>d3.js programming experience battleship program python te minal<jupyter_text>Vectorize Speeches Use spacy's inbuilt embedding model to vectorize our speeches<jupyter_code># Each speech is stored as a list of paragraph strings. 
# Here we join the paragraphs into a single speech string
speech_list = [" ".join(speech) for speech in speeches_pd['speech'].tolist() ]

## Pre-process and tokenize our speeches
t0 = time.time()

# preprocess text of all speeches
speeches_cleaned = [ spacy_preprocess(speech) for speech in speech_list]

# vectorize and convert to numpy array
speeches_embed = np.array([nlp(speech).vector for speech in speeches_cleaned])

print('Preprocessing time elapsed: ' + str(time.time()-t0))<jupyter_output>Preprocessing time elapsed: 107.23779678344727<jupyter_text>Do a k-nearest neighbors search to find similar speeches<jupyter_code># Find the closest X points
# we will find all other points in the dataset
k_search_dist = len(speeches_embed)
t0 = time.time()

# Scale document vectors (maybe not necessary?)
#speechvec_scaler = StandardScaler() # initialize scaler
#speeches_scaled = speechvec_scaler.fit_transform(speeches_embed)

kn_model = NearestNeighbors()
kn_model.fit(speeches_embed)

# find X most similar speeches for each speech
# We add 1 to k since each speech will be most similar to itself (and we remove that result)
dist_speeches, sim_speeches = kn_model.kneighbors(speeches_embed,k_search_dist)

print('k-nearest search time elapsed: ' + str(time.time()-t0))
# Story nump arrays in pandas
dist_speeches_pd =pd.DataFrame(dist_speeches)
dist_speeches_pd.insert(0,'speech_num',speeches_pd.index)

sim_speeches_pd =pd.DataFrame(sim_speeches)
sim_speeches_pd.insert(0,'speech_num',speeches_pd.index)
dist_matrix = pd.melt(dist_speeches_pd,
    id_vars=['speech_num'],value_vars=list(range(0,k_search_dist))).\
    rename({'variable':'rank','value': 'distance'},axis='columns')

sim_matrix = pd.melt(sim_speeches_pd,
    id_vars=['speech_num'],value_vars=list(range(0,k_search_dist))).\
    rename({'variable': 'rank','value':'speech_num_match'},axis='columns')<jupyter_output><empty_output><jupyter_text>Show the most 'similar' state of the union speeches according to spacy document embeddings<jupyter_code># Only keep one unique pair of matches and don't keep rows that match the same speech to itself
simdist_matrix = sim_matrix[(sim_matrix['speech_num'] != sim_matrix['speech_num_match']) & \
                        (sim_matrix['speech_num'] < sim_matrix['speech_num_match'])].\
    merge(dist_matrix,on=['speech_num','rank']).\
    merge(speeches_pd[['president','year']],left_on='speech_num',right_index=True).\
    merge(speeches_pd[['president','year']],left_on='speech_num_match',right_index=True,suffixes=['','_match']).\
    sort_values('distance')<jupyter_output><empty_output><jupyter_text>The most similar speeches<jupyter_code>simdist_matrix.head(5)<jupyter_output><empty_output><jupyter_text>Now let's eliminate cases where the president is the same<jupyter_code>simdist_matrix[simdist_matrix['president'] != simdist_matrix['president_match']].head(5)<jupyter_output><empty_output><jupyter_text>Most similar speeches to speeches occuring since 1950<jupyter_code>simdist_matrix[(simdist_matrix['president'] != simdist_matrix['president_match']) & \
              ((simdist_matrix['year'] >= 1950 ))].head(10).\
            drop(['speech_num','speech_num_match'],axis=1)<jupyter_output><empty_output><jupyter_text>Most dissimilar speeches since 1900. Note 1943 and 1944 by FDR are very dissimilar.<jupyter_code>simdist_matrix[((simdist_matrix['year'] >= 1900 ) & (simdist_matrix['year_match'] >= 1900 ))].\
                sort_values('distance',ascending=False).head(10).\
            drop(['speech_num','speech_num_match'],axis=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11549232664949032, 'fraction_numerical': 0.010529853254172734, 'mean_word_length': 5.1871101871101875, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 44, 'https://': 5, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '102503', 'license': 'MIT', 'n_tokens_mistral': 3030, 'n_tokens_neox': 2802, 'n_words': 724, 'path': 'Python/state_of_union_embeddings.ipynb', 'repo_name': 'Mustapha-Wasseja/Data-Science-Codex', 'size': '8927'}