<jupyter_start><jupyter_text>Project 4: Candidate Resume Recommendation System Import packages<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD, NMF, LatentDirichletAllocation
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
from sklearn.metrics import pairwise_distances
from sklearn.cluster import KMeans, DBSCAN
import pyLDAvis
import pyLDAvis.sklearn
import statistics 
import matplotlib.pyplot as plt
import seaborn as sns
import umap
import spacy
import en_core_web_sm
nlp = en_core_web_sm.load()<jupyter_output>/Users/hellojenny/opt/anaconda3/envs/metis/lib/python3.7/site-packages/thinc/neural/train.py:3: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
  from .optimizers import Adam, SGD, linear_decay
/Users/hellojenny/opt/anaconda3/envs/metis/lib/python3.7/site-packages/thinc/check.py:1: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
  from collections import defaultdict, Sequence, Sized, Iterable, Callable
/Users/hellojenny/opt/anaconda3/envs/metis/lib/python3.7/site-packages/thinc/check.py:1: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
  from collections import defaultdict, Sequence, Sized, Iterable, Callable
/Users/hellojenny/opt/anaconda3/en[...]<jupyter_text>Load the data into dataframes<jupyter_code>jobs = pd.read_pickle('./jobs_cleaned.pkl')
resumes = pd.read_pickle('./resumes_cleaned.pkl')<jupyter_output>/Users/hellojenny/opt/anaconda3/envs/metis/lib/python3.7/site-packages/botocore/vendored/requests/packages/urllib3/_collections.py:1: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
  from collections import Mapping, MutableMapping
/Users/hellojenny/opt/anaconda3/envs/metis/lib/python3.7/site-packages/botocore/vendored/requests/packages/urllib3/_collections.py:1: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
  from collections import Mapping, MutableMapping<jupyter_text>Conduct Additional EDASince there are only 2,000 rows of resumes and a lot of them have very little text after pre-processing, i.e., less than 20 words. I'll be cutting down the data to only include more than 23 words, as it will reserve at least 1,000 resumes for the recommendation engine.<jupyter_code>statistics.mode([len(text) for text in resumes['text']])
dist = [len(text) for text in resumes['text']]
plt.hist(dist, color = 'blue', edgecolor = 'black')
def display_cluster(X,km=[],num_clusters=0):
    color = 'brgcmyk'
    alpha = 0.5
    s = 20
    if num_clusters == 0:
        plt.scatter(X[:,0],X[:,1],c = color[0],alpha = alpha,s = s)
    else:
        for i in range(num_clusters):
            plt.scatter(X[km.labels_==i,0],X[km.labels_==i,1],c = color[i],alpha = alpha,s=s)
            plt.scatter(km.cluster_centers_[i][0],km.cluster_centers_[i][1],c = color[i], marker = 'x', s = 100)
X = [[len(text.split()), two] for text in resumes['text'] for two in [2]]
X = np.asarray(X)
X
num_clusters = 3
km = KMeans(n_clusters=num_clusters,random_state=10,n_init=10) # n_init, number of times the K-mean algorithm will run
km.fit(X)
display_cluster(X,km,num_clusters)
count = 0
for text in resumes['text']:
    if len(text.split()) > 23:
        count+=1
count
def num_words(text):
    return len(text.split())

resumes = resumes[resumes['text'].map(num_words) >= 23] 
resumes.reset_index(inplace=True)
resumes.drop(['index'], axis=1, inplace=True)
resumes<jupyter_output><empty_output><jupyter_text>Vectorizer Fit the vectorizer on `Resume`<jupyter_code>tfidf = TfidfVectorizer(ngram_range=(2,3), 
                        stop_words='english', 
                        sublinear_tf=True,
                        min_df=12, 
                        max_df=0.7)
doc_word = tfidf.fit_transform(resumes['text'])
doc_word.shape<jupyter_output><empty_output><jupyter_text>Dimension Reduction<jupyter_code>def display_topics(model, feature_names, no_top_words, topic_names=None):
    for ix, topic in enumerate(model.components_):
        if not topic_names or not topic_names[ix]:
            print("\nTopic ", ix+1)
        else:
            print("\nTopic: '",topic_names[ix],"'")
        print(", ".join([feature_names[i]
                        for i in topic.argsort()[:-no_top_words - 1:-1]]))
component_count = 8
component_names = [f'Topic {component_num+1}' for component_num in range(component_count)]<jupyter_output><empty_output><jupyter_text>LSA<jupyter_code>lsa = TruncatedSVD(component_count)
doc_topic = lsa.fit_transform(doc_word)

topic_word = pd.DataFrame(lsa.components_.round(3), 
                          index=component_names,
                          columns=tfidf.get_feature_names())

resume_doc_topic_lsa = pd.DataFrame(doc_topic.round(5), 
                                    index=resumes['text'], 
                                    columns = component_names)

print('Total Explained Variance Ratio:', lsa.explained_variance_ratio_.sum())
display_topics(lsa, tfidf.get_feature_names(), 10)<jupyter_output>Topic  1
customer service, sale associate, service representative, excellent customer, excellent customer service, cash register, customer service representative, train new, assist customer, social medium

Topic  2
social medium, administrative assistant, team member, day day, datum entry, business development, policy procedure, new employee, train new, sale associate

Topic  3
sale associate, cash register, answer phone, greet customer, answer question, help customer, train new, new employee, train new employee, datum entry

Topic  4
sale associate, social medium, help customer, business development, marketing strategy, real estate, sale marketing, team member, new business, marketing campaign

Topic  5
train new, new employee, train new employee, team member, account manager, cash register, excellent customer service, excellent customer, training new, human resources

Topic  6
social medium, train new, answer phone, new employee, train new employee, answer question, excellent custom[...]<jupyter_text>**Explained Variance**<jupyter_code>var_ratio = lsa.explained_variance_ratio_
num_components = [i+1 for i in range(len(var_ratio))]

plt.figure(figsize=(18, 6))
plt.subplot(1, 2, 1)
sns.lineplot(num_components, var_ratio)
sns.scatterplot(num_components, var_ratio)
plt.title('Explained Variance by LSA', fontsize=16, y= 1.03)
plt.xlabel('Topic Number', fontsize = 12, x=1.0)
plt.ylabel('Explained\nVariance', fontsize = 12, y=1.03, rotation=0)

plt.subplot(1, 2, 2)
cum_variance = np.cumsum(var_ratio)
sns.lineplot(num_components, cum_variance)
sns.scatterplot(num_components, cum_variance)
plt.title('Cumulative Explained Variance by LSA', fontsize=16, y= 1.03)
plt.xlabel('Number of Topics', fontsize = 12, x=1.01)
plt.ylabel('Cumulative \n Explained Variance', fontsize = 12, y=1.03, rotation=0)
# plt.savefig('./img/variance_per_component_lsa.png', bbox_inches = 'tight')<jupyter_output><empty_output><jupyter_text>NMF<jupyter_code>nmf = NMF(component_count, random_state=42)
doc_topic = nmf.fit_transform(doc_word)

topic_word = pd.DataFrame(nmf.components_.round(3),
                          index = component_names,
                          columns = tfidf.get_feature_names())

resume_doc_topic_nmf = pd.DataFrame(doc_topic.round(5), 
                                    index=resumes['text'], 
                                    columns = component_names)
display_topics(nmf, tfidf.get_feature_names(), 10)<jupyter_output>Topic  1
customer service, service representative, customer service representative, provide customer, provide customer service, assist customer, resolve customer, fast pace, open close, service skill

Topic  2
team member, policy procedure, business development, project manager, new business, responsibility include, day day, credit card, sale team, account manager

Topic  3
sale associate, help customer, cash register, greet customer, sale floor, provide customer, cash drawer, provide customer service, customer need, service customer

Topic  4
answer phone, administrative assistant, datum entry, data entry, line phone, clerical duty, multi line, order supply, travel arrangement, duty include

Topic  5
train new, new employee, train new employee, human resources, training new, executive assistant, inbound outbound, cash register, administrative support, service representative

Topic  6
social medium, marketing strategy, press release, act liaison, marketing campaign, small business, sp[...]<jupyter_text>LDA<jupyter_code>lda = LatentDirichletAllocation(n_components=component_count, # Number of topics
                                learning_method = 'online',
                                random_state=0,       
                                n_jobs = -1)  # Use all available CPUs

doc_topic = lda.fit_transform(doc_word)

topic_word = pd.DataFrame(lda.components_.round(3),
                          index = component_names,
                          columns = tfidf.get_feature_names())

resume_doc_topic_lda = pd.DataFrame(doc_topic.round(5), 
                                    index=resumes['text'], 
                                    columns = component_names)
#resume_doc_topic_lda
display_topics(lda, tfidf.get_feature_names(), 10)<jupyter_output>Topic  1
customer service, technical support, quality assurance, fast pace, security guard, high quality, sale representative, customer support, customer relation, provide customer

Topic  2
customer service, human resources, responsibility include, data entry, security officer, policy procedure, credit card, service representative, daily activity, customer service representative

Topic  3
sale associate, customer service, answer question, make sure, cash register, excellent customer, excellent customer service, assist customer, provide customer, provide excellent

Topic  4
software engineer, sale goal, train new, phone email, manager manage, day operation, day day operation, hardware software, customer relation, increase sale

Topic  5
project include, public relation, administrative duty, high school, timely manner, travel arrangement, duty include, office assistant, assistant manager, press release

Topic  6
social medium, business development, financial service, service client, pr[...]<jupyter_text>Plots Intertopic Distance Map<jupyter_code>lda_vis = pyLDAvis.sklearn.prepare(lda, doc_word, tfidf, mds='tsne', R=10)
print(f'\n')
pyLDAvis.enable_notebook()

lda_vis
##pyLDAvis.show(lda_vis)<jupyter_output><empty_output><jupyter_text>UMAP Plot **LDA UMAP based on Euclidean Distance**<jupyter_code>## Kmean Clustering for labels
num_clusters = 8
km = KMeans(n_clusters=num_clusters,random_state=10,n_init=1000) # n_init, number of times the K-mean algorithm will run
km.fit(resume_doc_topic_lda)
km.cluster_centers_
km_labels = km.labels_ + 1 # topic 1 ~ 8 instead of 0 ~ 7
labels = []
for label in km_labels:
    labels.append("Topic " + str(label))
    
## UMAP
reducer_euclidean = umap.UMAP(random_state = 42, n_components = 8)
reducer_euclidean.fit(resume_doc_topic_lda.values)
embeddings_euclidean = reducer_euclidean.transform(resume_doc_topic_lda.values)

## Plot
plt.figure(figsize=(10, 7))

sns.scatterplot(embeddings_euclidean[:, 0], 
                embeddings_euclidean[:, 1], 
                hue = labels,
                legend = 'full',
                palette = sns.color_palette("hls", 8),
                alpha = 0.5)

plt.title(f'\nUMAP Projection of Types of Resumes\n(LDA | Euclidean Distance)', fontsize = 20, y = 1.01);<jupyter_output><empty_output><jupyter_text>**NMF UMAP based on Cosine Similarity**<jupyter_code>## labels are the topic number with the max value
labels = resume_doc_topic_nmf.idxmax(axis=1).tolist()

## UMAP
reducer_cosine = umap.UMAP(random_state = 42, n_components = 8, metric='cosine', output_metric='cosine')
reducer_cosine.fit(resume_doc_topic_nmf.values)
embeddings_cosine = reducer_cosine.transform(resume_doc_topic_nmf.values)

## Plot
plt.figure(figsize=(10, 7))

sns.scatterplot(embeddings_cosine[:, 0], 
                embeddings_cosine[:, 1], 
                hue = labels,
                legend = 'full',
                palette = sns.color_palette("hls", 8),
                alpha = 0.5)

plt.title(f'\nUMAP Projection of Types of Resumes (NMF | Cosine Similarity)', fontsize = 20, y = 1.01);<jupyter_output><empty_output><jupyter_text>**LSA UMAP based on Cosine Similarity**<jupyter_code># labels are the topic number with the max value
labels = resume_doc_topic_lsa.idxmax(axis=1).tolist()

## UMAP
reducer_cosine = umap.UMAP(random_state = 42, n_components = 8, metric='cosine', output_metric='cosine')
reducer_cosine.fit(resume_doc_topic_lsa.values)
embeddings_cosine = reducer_cosine.transform(resume_doc_topic_lsa.values)

## Plot
plt.figure(figsize=(10, 7))

sns.scatterplot(embeddings_cosine[:, 0], 
                embeddings_cosine[:, 1], 
                hue = labels,
                legend = 'full',
                palette = sns.color_palette("hls", 8),
                alpha = 0.5)

plt.title(f'\nUMAP Projection of Types of Resumes (LSA | Cosine Similarity)', fontsize = 20, y = 1.01);<jupyter_output><empty_output><jupyter_text>Recommendation Engine **Recommendation function using `pairwise_distances`**<jupyter_code>def pairwise_func(text): 
    vt = tfidf.transform(sample_jd)
    tt = lsa.transform(vt)
    cos = pairwise_distances(tt,resume_doc_topic_lsa,metric='cosine').argsort()[0]
    doc_num_1 = cos[0]
    doc_num_2 = cos[1]
    print(list(cos[0:10]))
    print(resumes.iloc[doc_num_1, [2]])
    print(resumes.iloc[doc_num_2, [2]])<jupyter_output><empty_output><jupyter_text>**Recommendation function with `np.dot`**Below are the text cleaning functions from previous notebooks.<jupyter_code>def ultimate_cleaning(text):
    doc = nlp(text)
    pos_tag_list = ['ADJ', 'ADV', 'NOUN', 'PART', 'VERB'] 
    stop_words_list = ['nbsp', 'candidate', 'now', 'work', 'professional', 'ability', 'hire', 'join', 
                       'interested', 'approximately', 'start', 'end', 'p.m.', '-PRON-']
    cleaned_text = []

    for token in doc:
        if token.pos_ in pos_tag_list\
        and token.is_stop == False\
        and token.is_alpha == True\
        and token.lemma_ not in stop_words_list:
            cleaned_text.append(token.lemma_)

    return ' '.join(cleaned_text)
                    
def title_cleaning(text):
    doc = nlp(text)
    pos_tag_list = ['ADJ', 'ADV', 'NOUN', 'PART', 'VERB', 'PROPN'] 
    stop_words_list = ['nbsp', 'candidate', 'now', 'work', 'professional', 'ability', 'hire', 'join', 
                       'interested', 'approximately', 'start', 'end', 'p.m.', '-PRON-']
    cleaned_text = []

    for token in doc:
        if token.pos_ in pos_tag_list\
        and token.is_stop == False\
        and token.is_alpha == True\
        and token.lemma_ not in stop_words_list:
            cleaned_text.append(token.lemma_)

    return ' '.join(cleaned_text)
def resume_recommendation(model, job_description, resume_doc_topic_matrix, num_recom=2):
    
    jd_doc_term_vector = tfidf.transform(job_description)
    jd_doc_topic_vector = model.transform(jd_doc_term_vector)
    
    recs = []
    resume_doc_count = range(resume_doc_topic_matrix.shape[0])
    
    for doc in resume_doc_count:
        cosine_simliarity_score = np.dot(jd_doc_topic_vector, resume_doc_topic_matrix[doc])
        recs.append([doc, cosine_simliarity_score])
    
    final_rec = sorted(recs, key = lambda x: x[1],reverse=True)
    doc_num_1 = final_rec[0][0]
    doc_num_2 = final_rec[1][0]
    print(resumes.iloc[doc_num_1, [2]])
    print(resumes.iloc[doc_num_2, [2]])
    return final_rec[:num_recom]
job_title = "Retail Sales Associate"
job_description = f"""As a Retail Sales Associate, you’ll provide exceptional customer service and have our customer’s needs in mind while helping them find products and providing answers & solutions.

You will be positive, inclusive and collaborative in helping our customers. You will also be part of a fun, team-oriented retail culture at Staples.

Provide an exceptional customer experience.
• Respond quickly and resourcefully to customer requests and concerns
• Create a positive, inviting environment for customers

• Multitask on sales responsibilities and merchandising while maintaining a neat and clean store
• Understand and use basic selling skills to engage and present solutions
• Be flexible in performing various responsibilities (i.e. cashier, returns, reserving online pick-ups, other duties as assigned)
• Customer service or cashier experience in a retail environment """

job_title = title_cleaning(job_title)
job_description = ultimate_cleaning(job_description)
sample_jd = [job_title + " " + job_description]

resume_recommendation(lsa, sample_jd, resume_doc_topic_lsa.to_numpy(), num_recom=2)<jupyter_output>job_description    Customer service with footwear/keyholder custo...
Name: 336, dtype: object
job_description    Greeted resident at front desk,answer incoming...
Name: 517, dtype: object<jupyter_text>**Create a dataframe with recommended resume**<jupyter_code>vt = tfidf.transform(jobs['text'])
tt = lsa.transform(vt)
recs = pairwise_distances(tt,resume_doc_topic_lsa,metric='cosine').argsort()

from collections import defaultdict
top_rec_app_ids = []
top_rec_app_resumes = []
top_10_rec_app_ids = []

for rec in recs:
    doc_num = rec[0]
    top_rec_app_ids.append(resumes['applicant_id'].loc[doc_num])
    top_rec_app_resumes.append(resumes['text'].loc[doc_num])
    top_10_rec_app_ids.append(list(rec[0:10]))
    
rec_dict = {'job_id': jobs['job_id'].tolist(), 
            'job_description_cleaned': jobs['text'].tolist(),
            'top_rec_app_id': top_rec_app_ids, 
            'top_rec_app_resume_cleaned': top_rec_app_resumes, 
            'top_10_rec_app_ids': top_10_rec_app_ids}
rec_df = pd.DataFrame.from_dict(rec_dict)
rec_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10443312378061999, 'fraction_numerical': 0.015120312161283329, 'mean_word_length': 4.293459552495697, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '527713', 'license': 'MIT', 'n_tokens_mistral': 6107, 'n_tokens_neox': 5828, 'n_words': 1507, 'path': '03 Topic Modeling & Recommendation Engine.ipynb', 'repo_name': 'WorqHat/Resume-Recommendation', 'size': '18452'}