<jupyter_start><jupyter_text>Load Resume<jupyter_code>with open('BryantBiggs_mod.txt', 'r') as f:
    new_resume = f.read().strip()
new_resume<jupyter_output><empty_output><jupyter_text>City, State Abbreviation List<jupyter_code>with open(r'pkl/cities.pkl', 'rb') as infile:
       cities = pickle.load(infile)

with open(r'pkl/abbr.pkl', 'rb') as infile:
       abbr = pickle.load(infile)<jupyter_output><empty_output><jupyter_text>Clean Text<jupyter_code># remove special characters
new_resume = re.sub(r'([-/*&%+();?.,@#$^])', ' ', new_resume)

cleaner = ['\xa0']
for c in cleaner:
    new_resume = new_resume.replace(c,'')

# remove multiple white spaces
new_resume = re.sub('\n',' ', new_resume)
new_resume = re.sub('\t',' ', new_resume)
new_resume = re.sub(r'''[^0-9a-zA-Z ]+''', '', new_resume)
new_resume = re.sub(' +',' ', new_resume)
new_resume<jupyter_output><empty_output><jupyter_text>Remove StopWords<jupyter_code># cache stopwords first to reduce compute time
cachedStopWords = stopwords.words("english")
cachedStopWords += ['tot']
cachedStopWords += cities
cachedStopWords += abbr
cachedStopWords = list(set(cachedStopWords))
# convert all text to lower case and separate into list
res_text = [x for x in new_resume.lower().split()]

# remove stopwords
res_text = ' '.join([item for item in res_text if item not in cachedStopWords])
print(len(res_text.split()))
res_text<jupyter_output><empty_output><jupyter_text>More Cleaning (Ugly - Fix Me)<jupyter_code>res_text = res_text.replace('aacademic', 'academic')
res_text = res_text.replace('aaccounts', 'account')
res_text = res_text.replace('aaerobic', 'aerobic')
res_text = res_text.replace('aademy', 'academy')

res_text = res_text.replace('aanalyst', 'analyst')
res_text = res_text.replace('aanalyzed', 'analyst')

res_text = res_text.replace('reports', 'report')
res_text = res_text.replace('reporting', 'report')

res_text = res_text.replace('engineering', 'engineer')
res_text = res_text.replace('engineers', 'engineer')

res_text = res_text.replace('services', 'service')
res_text = res_text.replace('servicing', 'service')

res_text = res_text.replace('systems', 'system')

res_text = res_text.replace('abandoned', 'abandon')
res_text = res_text.replace('abandoner', 'abandon')
res_text = res_text.replace('abandoning', 'abandon')
res_text = res_text.replace('abandonment', 'abandon')
res_text = res_text.replace('abandons', 'abandon')

res_text = res_text.replace('management', 'manage')
res_text = res_text.replace('manager', 'manage')

res_text = res_text.replace('development', 'develop')
res_text = res_text.replace('developer', 'develop')

print(len(res_text.split()))
res_text<jupyter_output>300<jupyter_text>Word Counts<jupyter_code>def get_wordcount(text_list, min_ct=3, most_common=30, get_all=False):
    '''
    returns => most common
    '''
    # get wordcount counter object
    word_count = Counter(text_list)

    # remove words that occur min_ct times or less
    word_count = Counter({k:v for k, v in word_count.items() if v >= min_ct})

    if get_all:
        # return all
        word_count = word_count.items()
    else:
        # limit wordcounts for visualization
        word_count = word_count.most_common(most_common)
    
    return word_count<jupyter_output><empty_output><jupyter_text>Extrack Noun Phrases<jupyter_code>res_nouns = TextBlob(res_text).noun_phrases
res_nouns<jupyter_output><empty_output><jupyter_text>Merge Noun Phrases Back to Text String<jupyter_code>temp = []
for i in res_nouns:
    temp.append(i)

res_nouns = ''
for x in temp:
    res_nouns += x + ' '
res_nouns<jupyter_output><empty_output><jupyter_text>Tokenize the documents, remove stop words and words that only appear once<jupyter_code># remove common words and tokenize
stoplist = set('for a of the and to in'.split())
texts = [word for word in res_nouns.split()]

# remove words that appear only once
frequency = defaultdict(int)
for text in texts:
        frequency[text] += 1

# remove words that occur less than n times
texts = [token for token in texts if frequency[token] > 1]
str_text = ' '.join(texts)<jupyter_output><empty_output><jupyter_text>Load Transformation Interface<jupyter_code># load sparse vector matrix
if (os.path.exists('pkl/resume_token.mm')):
    corpus = corpora.MmCorpus('pkl/resume_token.mm')
    print('Sparse matrix LOADED as \'corpus\'')
else:
    print('Sparse matrix NOT FOUND')
# load tokenized dictionary
if (os.path.exists('pkl/resume_token.dict')):
    dictionary = corpora.Dictionary.load('pkl/resume_token.dict')
    print('Tokenized dictionary LOADED as \'dictionary\'')
else:
    print('Tokenized dictionary NOT FOUND')<jupyter_output>Tokenized dictionary LOADED as 'dictionary'<jupyter_text>Load Trained Model<jupyter_code>lsi_mdl = models.LsiModel.load('pkl/lsi_mdl.lsi')
vec_bow = dictionary.doc2bow(texts)
vec_lsi = lsi_mdl[vec_bow] # convert the query to LSI space
print(vec_lsi)
index = similarities.MatrixSimilarity(lsi_mdl[corpus])
# perform a similarity query against the corpus
sims = index[vec_lsi] 

# (document_number, document_similarity)
sim_lst = list(enumerate(sims))

# sort relevant documents first
sim_lst.sort(key=operator.itemgetter(1), reverse=True)
sim_lst[1:6]
df = pd.read_pickle('pkl/df_stop_noun.pkl')
df.iloc[14945]['resume_nouns']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13664596273291926, 'fraction_numerical': 0.003764351590438547, 'mean_word_length': 6.190798376184032, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '597255', 'license': 'MIT', 'n_tokens_mistral': 1946, 'n_tokens_neox': 1916, 'n_words': 390, 'path': 'input.ipynb', 'repo_name': 'vanessanielsen/resume_tailor', 'size': '5313'}