<jupyter_start><jupyter_text>Vector Spaces<jupyter_code>import logging
#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

import gensim
from gensim import corpora, models, similarities
from nltk.corpus import stopwords

from collections import defaultdict
from pprint import pprint
from six import iteritems
import os

import numpy as np
import pandas as pd
import scipy.sparse<jupyter_output><empty_output><jupyter_text>Load Processed Dataframe<jupyter_code>df = pd.read_pickle('pkl/df_stop_noun.pkl')
df.head(3)<jupyter_output><empty_output><jupyter_text>Convert Series to List of Strings<jupyter_code>resumes = df['resume_nouns'].tolist()
resumes[:1]<jupyter_output><empty_output><jupyter_text>From Strings to Vectors  Tokenize the documents, remove stop words and words that only appear once<jupyter_code># remove common words and tokenize
stoplist = set('for a of the and to in'.split())
texts = [[word for word in resume.split()] for resume in resumes]

# remove words that appear only once
frequency = defaultdict(int)
for text in texts:
    for token in text:
        frequency[token] += 1

# remove words that occur less than n times
texts = [[token for token in text if frequency[token] > 2] for text in texts]<jupyter_output><empty_output><jupyter_text>Save Token Count Dictionary to File<jupyter_code>dictionary = corpora.Dictionary(texts)

# store the dictionary, for future reference
dictionary.save('pkl/resume_token.dict')
print(dictionary)<jupyter_output>Dictionary(42606 unique tokens: ['blog', 'dtac', 'melmark', 'ravishankar', 'plate']...)<jupyter_text>Convert Tokenized Resumes to Vectors<jupyter_code>corpus = [dictionary.doc2bow(text) for text in texts]
corpora.MmCorpus.serialize('pkl/resume_token.mm', corpus)  # store to disk, for later use
for c in corpus[:1]:
    print(c)<jupyter_output>[(0, 1), (1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 2), (8, 1), (9, 1), (10, 1), (11, 1), (12, 1), (13, 1), (14, 1), (15, 1), (16, 1), (17, 1), (18, 1), (19, 1), (20, 1), (21, 1), (22, 1), (23, 8), (24, 2), (25, 1), (26, 1), (27, 1), (28, 2), (29, 1), (30, 1), (31, 2), (32, 1), (33, 1), (34, 1), (35, 1), (36, 1), (37, 1), (38, 1), (39, 1), (40, 1), (41, 1), (42, 1), (43, 1), (44, 1), (45, 1), (46, 1), (47, 1), (48, 1), (49, 1), (50, 1), (51, 1), (52, 1), (53, 1), (54, 1), (55, 1), (56, 1), (57, 2)]<jupyter_text>Corpus Streaming – One Document at a Time<jupyter_code># replace 'texts' with 'open(my_file.txt)' to read from files (one line in the file is a document)
# or loop through and open each individual file (?)
# either way, dictionary.doc2bow wants a list of words (aka - line.lower().split())
class MyCorpus(object):
    def __iter__(self):
        for line in texts:
            yield dictionary.doc2bow(line)
# doesn't load the corpus into memory!
corpus_memory_friendly = MyCorpus()<jupyter_output><empty_output><jupyter_text>Similarly, to construct the dictionary without loading all texts into memory<jupyter_code>_ = '''
# collect statistics about all tokens
dictionary = corpora.Dictionary(line.lower().split() for line in open('mycorpus.txt'))

# remove stop words and words that appear only once
stop_ids = [dictionary.token2id[stopword] for stopword in stoplist 
            if stopword in dictionary.token2id]
once_ids = [tokenid for tokenid, docfreq in iteritems(dictionary.dfs) if docfreq == 1]

# remove stop words and words that appear only once
dictionary.filter_tokens(stop_ids + once_ids)

# remove gaps in id sequence after words that were removed
dictionary.compactify()
print(dictionary)
'''<jupyter_output><empty_output><jupyter_text>Transformation Interface<jupyter_code># load tokenized dictionary
if (os.path.exists('pkl/resume_token.dict')):
    dictionary = corpora.Dictionary.load('pkl/resume_token.dict')
    print('Tokenized dictionary LOADED as \'dictionary\'')
else:
    print('Tokenized dictionary NOT FOUND')
# load sparse vector matrix
if (os.path.exists('pkl/resume_token.mm')):
    corpus = corpora.MmCorpus('pkl/resume_token.mm')
    print('Sparse matrix LOADED as \'corpus\'')
else:
    print('Sparse matrix NOT FOUND')<jupyter_output>Sparse matrix LOADED as 'corpus'<jupyter_text>TF-IDF Transformation<jupyter_code># step 1 -- initialize a model
tfidf_mdl = models.TfidfModel(corpus)<jupyter_output><empty_output><jupyter_text>Calling `model[corpus]` only creates a wrapper around the old corpus document stream – actual conversions are done on-the-fly, during document iteration. We cannot convert the entire corpus at the time of calling corpus_transformed = model[corpus], because that would mean storing the result in main memory, and that contradicts gensim’s objective of memory-indepedence. If you will be iterating over the transformed corpus_transformed multiple times, and the transformation is costly, serialize the resulting corpus to disk first and continue using that.<jupyter_code># step 2 -- use the model to transform vectors
corpus_tfidf = tfidf_mdl[corpus]

# view one resume
for doc in corpus_tfidf[:1]:
    print(doc)
from sklearn.feature_extraction.text import TfidfVectorizer
n_features = 1000

tfidf_vec = TfidfVectorizer(input='content', ngram_range=(1, 3), max_df=0.9, min_df=2, 
                max_features=n_features, norm='l2', use_idf=True, smooth_idf=True, sublinear_tf=False)

tfidf_vec_prep = tfidf_vec.fit_transform(resumes)
from sklearn.cluster import KMeans
from sklearn import metrics

km = KMeans(n_clusters=8, init='k-means++', max_iter=100, n_init=1)

km_mdl = km.fit_predict(tfidf_vec_prep)
# Determine your k range
k_range = range(1,20)

# fit the kmeans model for each n_clusters = k
k_means_var = [KMeans(n_clusters=k).fit(tfidf_vec_prep) for k in k_range]

# pull out the cluster centers for each model
centroids = [X.cluster_centers_ for X in k_means_var]
from scipy.spatial.distance import cdist, pdist

# calculate the euclidean distance from each point to each cluster center
k_euclid = [cdist(tfidf_vec_prep.toarray(), cent, 'euclidean') for cent in centroids]
dist = [np.min(ke, axis=1) for ke in k_euclid]

# total within-cluster sum of squares
wcss = [sum(d**2) for d in dist]

# the total sum of squares
tss = sum(pdist(tfidf_vec_prep.toarray())**2)/tfidf_vec_prep.shape[1]

# the between-cluster sum of squares
bss = tss - wcss
import numpy as np
from scipy.cluster.vq import kmeans,vq
from scipy.spatial.distance import cdist
import matplotlib.pyplot as plt

##### cluster data into K=1..10 clusters #####
K = range(1,20)

# scipy.cluster.vq.kmeans
KM = [kmeans(tfidf_vec_prep.toarray(),k) for k in K]
centroids = [cent for (cent,var) in KM]   # cluster centroids

# alternative: scipy.spatial.distance.cdist
D_k = [cdist(tfidf_vec_prep.toarray(), cent, 'euclidean') for cent in centroids]
cIdx = [np.argmin(D,axis=1) for D in D_k]
dist = [np.min(D,axis=1) for D in D_k]
avgWithinSS = [sum(d)/tfidf_vec_prep.shape[0] for d in dist]

##### plot ###
kIdx = 2

# elbow curve
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(K, avgWithinSS, 'b*-')
import seaborn as sns
sns.set_style("white")
sns.set_context("poster", font_scale=1.25, rc={"lines.linewidth": 2.5})
sns.set_palette("Set2")
colors = sns.color_palette("BrBG", 5)
# make figure
fig = plt.figure(figsize=(20,12))
ax = fig.add_subplot(111)

# color
colors = sns.color_palette("BrBG", 10)

# plots
ax.plot(K, avgWithinSS, marker='o', color=colors[-1], alpha=0.5)

# labels/titles
plt.legend(loc="best")
plt.title('Elbow for K-Means')
plt.xlabel('Number of Clusters')
plt.ylabel('Avg. Within-Cluster Sum of Squares')

# remove border
ax.spines["top"].set_visible(False)
ax.spines["bottom"].set_visible(False)
ax.spines["right"].set_visible(False)
ax.spines["left"].set_visible(False)

# show grid
ax.xaxis.grid(True, alpha=0.2) 
ax.yaxis.grid(True, alpha=0.2) 

# plot that biddy
plt.savefig('data/pics/{0}.png'.format('KMeans_elbow'), bbox_inches='tight')
plt.close(fig)
import numpy as np
from scipy.cluster.vq import kmeans
from scipy.spatial.distance import cdist,pdist
from sklearn import datasets
from sklearn.decomposition import RandomizedPCA
from matplotlib import pyplot as plt
from matplotlib import cm

# perform PCA dimensionality reduction
pca = RandomizedPCA(n_components=2).fit(tfidf_vec_prep.toarray())
X = pca.transform(tfidf_vec_prep.toarray())

##### cluster data into K=1..20 clusters #####
K_MAX = 20
KK = range(1,K_MAX+1)

KM = [kmeans(X,k) for k in KK]
centroids = [cent for (cent,var) in KM]
D_k = [cdist(X, cent, 'euclidean') for cent in centroids]
cIdx = [np.argmin(D,axis=1) for D in D_k]
dist = [np.min(D,axis=1) for D in D_k]

tot_withinss = [sum(d**2) for d in dist]  # Total within-cluster sum of squares
totss = sum(pdist(X)**2)/X.shape[0]       # The total sum of squares
betweenss = totss - tot_withinss          # The between-cluster sum of squares

##### plots #####
kIdx = 4        # K=10
clr = cm.spectral( np.linspace(0,1,10) ).tolist()
mrk = 'os^p<dvh8>+x.'
# make figure
fig = plt.figure(figsize=(20,12))
ax = fig.add_subplot(111)

# color
colors = sns.color_palette("BrBG", 5)

# plots
#ax.plot(K, avgWithinSS, marker='o', color=colors[-1], alpha=0.5)
ax.plot(KK, betweenss/totss*100, marker='o', color=colors[-1], alpha=0.5)
ax.plot(KK[kIdx], betweenss[kIdx]/totss*100, marker='o', markersize=25, color=colors[0], alpha=0.5)

# labels/titles
plt.legend(loc="best")
plt.title('Elbow for KMeans Clustering')
plt.xlabel('Number of clusters')
plt.ylabel('Percentage of variance explained (%)')

ax.set_xlim((-0.1,20.5))
ax.set_ylim((-0.5,100))

# remove border
ax.spines["top"].set_visible(False)
ax.spines["bottom"].set_visible(False)
ax.spines["right"].set_visible(False)
ax.spines["left"].set_visible(False)

# show grid
ax.xaxis.grid(True, alpha=0.2) 
ax.yaxis.grid(True, alpha=0.2) 

# plot that biddy
plt.savefig('data/pics/{0}.png'.format('KMeans_elbow_var'), bbox_inches='tight')
plt.close(fig)
# make figure
fig = plt.figure(figsize=(20,12))
ax = fig.add_subplot(111)

# plots
for i in range(kIdx+1):
    ind = (cIdx[kIdx]==i)
    ax.scatter(X[ind,0],X[ind,1], s=65, c=colors[i], marker=mrk[i], 
               label='Cluster {0}'.format(i), alpha=1)

# labels/titles
plt.legend(loc='lower right')
plt.title('K={0} Clusters'.format(KK[kIdx]))

ax.set_xlim((-.5,.5))
ax.set_ylim((-.5,.5))

# remove border
ax.spines["top"].set_visible(False)
ax.spines["bottom"].set_visible(False)
ax.spines["right"].set_visible(False)
ax.spines["left"].set_visible(False)

# show grid
ax.xaxis.grid(True, alpha=0.2) 
ax.yaxis.grid(True, alpha=0.2) 

# plot that biddy
plt.savefig('data/pics/{0}.png'.format('KMeans_{0}_clusters'.format(KK[kIdx])), bbox_inches='tight')
plt.close(fig)
from sklearn.cluster import DBSCAN 
from sklearn.preprocessing import StandardScaler

dbscan = DBSCAN(eps=0.5, min_samples=5, metric='cosine', algorithm='brute', 
                leaf_size=30, p=None, random_state=None)

dbscan_mdl = dbscan.fit_predict(tfidf_vec_prep)<jupyter_output><empty_output><jupyter_text>Latent Semantic Indexing Topics<jupyter_code>num_topics = 100

# initialize an LSI transformation
lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=num_topics)
corpus_lsi = lsi[corpus_tfidf]
# the topics are printed to log
a = lsi.print_topics(8)
a[0]
for doc in corpus_lsi[800]: # both bow->tfidf and tfidf->lsi transformations are actually executed here, on the fly
    pass
    #print(doc)<jupyter_output><empty_output><jupyter_text>Model Save & Load<jupyter_code>lsi.save('pkl/lsi_mdl.lsi')
lsi = models.LsiModel.load('pkl/lsi_mdl.lsi')<jupyter_output><empty_output><jupyter_text>LDA Topics<jupyter_code>lda_mdl = models.LdaModel(corpus, id2word=dictionary, num_topics=20)
lda_mdl.top_topics
pprint(lda_mdl.print_topics(10))
print(corpus)
doc = df.iloc[0]['resume_nouns']
vec_bow = dictionary.doc2bow(doc.lower().split())
vec_lsi = lsi[vec_bow] # convert the query to LSI space
print(vec_lsi)<jupyter_output>[(0, 2.374975010869965), (1, 0.51728887522253952), (2, -0.058935199530753268), (3, 0.3578493537974749), (4, 1.560417648600577), (5, -1.9931029846659232), (6, 0.58697139609914861), (7, 1.437193124041608), (8, -0.38633595032575146), (9, -2.3068352804125016), (10, 0.77482570234627612), (11, -0.66082521176920128), (12, -2.0221618401059822), (13, 1.3229424544863675), (14, -0.29408524037515837), (15, -1.0569710323996966), (16, 1.110889840043604), (17, 1.3434022602282594), (18, -0.095802335904933394), (19, -0.80089048085959047), (20, -0.64832039201675884), (21, 1.35059095621303), (22, 0.36313071163680766), (23, 0.23008512654094881), (24, -1.4704302056681957), (25, -0.51110545886820391), (26, 1.5065962351771218), (27, -0.85864630999976976), (28, -0.27005311330166226), (29, 1.3357001963834654), (30, 0.11920370036201439), (31, 0.20935482520268536), (32, 0.58140672694418549), (33, 0.86476990150558442), (34, 0.21906262257842274), (35, 1.2623527033747142), (36, 0.47122700487966684), (37, 0.14754992[...]<jupyter_text>Cosine Similarity<jupyter_code>index = similarities.MatrixSimilarity(lsi[corpus]) # transform corpus to LSI space and index it
index.save('pkl/resume_stopped.index')
index = similarities.MatrixSimilarity.load('pkl/resume_stopped.index')
sims = index[vec_lsi] # perform a similarity query against the corpus

# (document_number, document_similarity)
sim_lst = list(enumerate(sims))
import operator
sim_lst.sort(key=operator.itemgetter(1), reverse=True)
# comparing resumes within resumes
sim_lst[1:6]
' '.join(texts[0])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14789778636161047, 'fraction_numerical': 0.07978012182439459, 'mean_word_length': 5.785786290322581, 'pattern_counts': {'":': 1, '<': 58, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '370441', 'license': 'MIT', 'n_tokens_mistral': 5845, 'n_tokens_neox': 5151, 'n_words': 1243, 'path': 'cosine.ipynb', 'repo_name': 'vanessanielsen/resume_tailor', 'size': '13462'}