<jupyter_start><jupyter_text>Word2Vec<jupyter_code>#we train our model now
corpus = df['title'].map( simple_preprocess)
#we chceck how it looks like
corpus
#now we transfer the 'corpus' to our model
model = Word2Vec(corpus, size=100, window=2, min_count=1)
model.wv.most_similar('machine')<jupyter_output>/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.
  if np.issubdtype(vec.dtype, np.int):<jupyter_text>Some example<jupyter_code>corpus = [
    ['machine', 'learning', 'c'],
    ['machine', 'learning', 'x'],
    ['y', 'machine', 'learning', 'w'],
    ['q', 'machine', 'learning', 'u', 'k'],
]
bigram = Phraser( Phrases(corpus, min_count=1, threshold=1))

bigram[ ['k', 'machine', 'learning', 'c']]<jupyter_output><empty_output><jupyter_text>Title + phrases<jupyter_code>title_corpus = df['title'].map(simple_preprocess)

title_bigram = Phraser( Phrases(title_corpus, min_count=1, threshold=1))
title_bigram[ simple_preprocess('Deep Learning Applaied Resarcher - Chicago')]
#we chceck the whole list now i.e. the whole phrase
title_corpus_phrase = [title_bigram[sent] for sent in title_corpus]
model = Word2Vec(title_corpus_phrase, size=100, window=2, min_count=1)
model.wv.most_similar('machine') #we check it
#above model is insufficient so we define below function
def prepare_corpus(corpus, bigram):
    for sent in corpus:
        yield bigram[sent] + sent
simple_preprocess('Deep Learning Applaied Resarcher - Chicago')
title_bigram[ ['deep', 'learning', 'appleid', 'resarcher', 'chicago']]
#we glue the both list together, i.e. bigram and preprocess
ext_corp = list(prepare_corpus(title_corpus, title_bigram))
title_model = Word2Vec( ext_corp, size=100, window=2, min_count=1 )
#now it works as it should
title_model.wv.most_similar('machine_learning')<jupyter_output>/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.
  if np.issubdtype(vec.dtype, np.int):<jupyter_text>Description<jupyter_code>simple_preprocess(df.sample()['description'].values[0])
#we do the same as above for bigram
descr_corpus = df['description'].map(simple_preprocess)
descr_bigram = Phraser(Phrases(descr_corpus, min_count=1, threshold=1))
ext_descr_corp = list(prepare_corpus(descr_corpus, descr_bigram))
descr_model = Word2Vec(ext_corp, size=100, window=2, min_count=1)
descr_model.wv.most_similar('nlp')
descr_model.wv.most_similar('pytorch')
df.sample()['description'].map(literal_eval).values
for line in df.sample()['description'].map(literal_eval).values[0]:
    print(line)
    print("")<jupyter_output>8 years of experience in the Information Technology 3 years of experience as a Senior Business Analyst Bachelor degree in business, information technology or related field Must have experience working on large scale projects Strong project management, ability to successfully manage multiple tasks at any given point, strong relationship building skills communication skills
{'fraction_non_alphanumeric': 0.12546125461254612, 'fraction_numerical': 0.015067650676506766, 'mean_word_length': 6.857487922705314, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '491457', 'license': 'MIT', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1087, 'n_words': 284, 'path': 'day4.ipynb', 'repo_name': 'beatass/DataWorkshop-challenges-4', 'size': '3252'}