<jupyter_start><jupyter_text>Word2Vec<jupyter_code>def process_sentence(sentence):
  new_tokens = word_tokenize(sentence)
  new_tokens = [t.lower() for t in new_tokens]
  new_tokens =[t for t in new_tokens if t not in stopwords.words('english')]
  new_tokens = [t for t in new_tokens if t.isalpha()]
  lemmatizer = WordNetLemmatizer()
  new_tokens =[lemmatizer.lemmatize(t) for t in new_tokens]
  return new_tokens
titles = df['title_orig'].tolist()
data = [process_sentence(t) for t in titles]<jupyter_output><empty_output><jupyter_text>Implemented based on official documentation: https://radimrehurek.com/gensim/models/word2vec.html.<jupyter_code>model = gensim.models.Word2Vec(sentences=data, size=100, window=5, min_count=1, workers=4)
model.train(data,total_examples=len(data),epochs=10)
print(model.wv['woman'])  # Find the vector of the word.
print(model.wv.most_similar('woman', topn=10))  # 10 Most similar words for inspection.
def caculate_average_word2vec(model, word_list):
  sum = np.matrix([model.wv[w] for w in word_list]).sum(axis=0)
  return sum / len(word_list)
word2vec_avg = [caculate_average_word2vec(model, d) for d in data]
df['word2vec_avg'] = word2vec_avg<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12458745874587458, 'fraction_numerical': 0.017326732673267328, 'mean_word_length': 6.980263157894737, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '435314', 'license': 'MIT', 'n_tokens_mistral': 465, 'n_tokens_neox': 450, 'n_words': 94, 'path': 'text_analysis.ipynb', 'repo_name': 'jqli0201/etsy-analysis', 'size': '1212'}