<jupyter_start><jupyter_text>cleaning<jupyter_code>def cleaning(doc):
    # Lemmatizes and removes stopwords
    # doc needs to be a spacy Doc object
    doc = re.sub("[^A-Za-z']+" , ' ' , doc).lower()
    doc = nlp(doc)
    txt = [token.lemma_ for token in doc if not token.is_stop]
    # Word2Vec uses context words to learn the vector representation of a target word,
    # if a sentence is only one or two words long,
    # the benefit for the training is very small
    if len(txt) > 2:
        return ' '.join(txt)
    
# brief_cleaning = (re.sub("[^A-Za-z']+", ' ', str(row)).lower() for row in df['text'])

print(cleaning('asa asas ass'))
#using spacy pipe (spacy.pipe())
# t = time.time()

# txt = [cleaning(doc) for doc in nlp.pipe(brief_cleaning , batch_size=5000 , n_threads=-1)]

# print(f'Time to clean up everything :{round(time.time() - t)/60,2}mins')
# df_clean = pd.DataFrame({'clean': txt})
# df_clean = df_clean.dropna().drop_duplicates()
# df_clean.shape
cleaned_text = df['text'].apply(cleaning)
df_clean = pd.DataFrame({'clean':cleaned_text})
df_clean.shape
df_clean = df_clean.dropna().drop_duplicates()
df_clean.shape
df_clean.head()<jupyter_output><empty_output><jupyter_text>bigrams<jupyter_code>from gensim.models.phrases import Phrases , Phraser
sent = [row.split() for row in df_clean['clean']]
sent[:5]
#now create relevant phrases from the list of sentences

phrases = Phrases(sent , min_count=30 , progress_per=10000)
phrases<jupyter_output><empty_output><jupyter_text>Most frequent words<jupyter_code>biagram_phraser = Phraser(phrases)
biagram_phraser
sentences = biagram_phraser[sent]
sentences.corpus[:10]
word_freq = defaultdict(int)

for sent in sentences:
    for i in sent:
        word_freq[i] +=1
len(word_freq)
word_freq
sorted(word_freq, key=word_freq.get, reverse=True)[:10]<jupyter_output><empty_output><jupyter_text>Train the model<jupyter_code>import multiprocessing

from gensim.models import Word2Vec
cores = multiprocessing.cpu_count() # Count the number of cores in a computer
cores
w2v_model = Word2Vec(min_count=20,
                     window=2,
                     size=300,
                     sample=6e-5, 
                     alpha=0.03, 
                     min_alpha=0.0007, 
                     negative=20,
                     workers=cores-1)
t = time.time()

w2v_model.build_vocab(sentences , progress_per=5000)

print(f'Time to build the vocab:  {round(time.time()-t)/60,2}')
t = time.time()

w2v_model.train(sentences, total_examples=w2v_model.corpus_count, epochs=30, report_delay=1)

print('Time to train the model: {} mins'.format(round((time.time() - t) / 60, 2)))


w2v_model.init_sims(replace=True)<jupyter_output><empty_output><jupyter_text>exploring the model<jupyter_code>w2v_model.wv.most_similar(positive=['virus'])
#biagram

w2v_model.wv.most_similar(positive=['corona_virus'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13070539419087138, 'fraction_numerical': 0.022130013831258646, 'mean_word_length': 4.269581056466302, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '385680', 'license': 'MIT', 'n_tokens_mistral': 1103, 'n_tokens_neox': 1044, 'n_words': 216, 'path': 'nlp-with-python/models/word2vec and gensim.ipynb', 'repo_name': 'xettrisomeman/practice', 'size': '2892'}