<jupyter_start><jupyter_text>Word representation using Scikit-learn<jupyter_code>import numpy as np

data = np.array([
    'this is the first document',
    'this document is the second document',
    'this is the third one not the first nor the third',
    'is this the first document or is is another document'
])

data
sentWords = [[word for word in sentence.split()] for sentence in data]

sentWords<jupyter_output><empty_output><jupyter_text>I. TF-IDF<jupyter_code># train a dictionary from a corpus
from gensim.corpora import Dictionary

dct = Dictionary(sentWords)  # fit dictionary

# (ID, count)
dct.doc2bow(["this", "is", "the", "document", "non_existent_word"])
from gensim.models import TfidfModel

corpus = [dct.doc2bow(doc) for doc in sentWords]  # convert corpus to BoW format
model = TfidfModel(corpus)  # fit model
model
vector = model[corpus[0]]  # apply model to the first corpus document
vector<jupyter_output><empty_output><jupyter_text>II. LSA<jupyter_code>from gensim.models import LsiModel

lsa_model = LsiModel(corpus, num_topics=3, id2word=dct)

lsa_model
# U: Terms X Concepts matrix 
lsa_model.projection.u
# S: Concepts X Concepts
lsa_model.projection.s
# D: Documents X Concepts
D = lsa_model[corpus]

for d in D:
    print(d)
# show the first topic
lsa_model.show_topic(0)<jupyter_output><empty_output><jupyter_text>III. LDA<jupyter_code>from gensim.models.ldamodel import LdaModel

lda = LdaModel(corpus=corpus, id2word=dct, num_topics=4, update_every=1, chunksize=100, passes=1)
lda
# use LDA model: transform new doc to bag-of-words, then apply lda
doc_bow = dct.doc2bow(["this", "is", "not", "the", "third", "second", "or", "third", "document"])
doc_bow
doc_lda = lda[doc_bow]
# doc_lda is vector of length num_topics representing weighted presence of each topic in the doc
doc_lda<jupyter_output><empty_output><jupyter_text>IV. Word2Vec<jupyter_code>from gensim.models import Word2Vec

model_w2v= Word2Vec(sentences=sentWords, vector_size=4, window=3, epochs=20, min_count=1)
word_vectors = model_w2v.wv

word_vectors
# vocabulary
word_vectors.key_to_index
# get embedding: method1
word_vectors["first"]
# get embedding: method2
word_vectors.get_vector("document")
# cosine similarity
word_vectors.distance("first","document")
# save the word2vec model
w2vpath = '/home/kariminf/Data/tutoriel/gensim_word2vec.model'
model_w2v.save(w2vpath)
# récupérer un modèle sauvegardé
model2 = Word2Vec.load(w2vpath)

# search the top 5 similar words to another
word = "another"
sim_words = []
if word in model2.wv: # tester si le mot existe dans la vocabulaire
    sim_words = model2.wv.most_similar(word, topn=5)
sim_words<jupyter_output><empty_output><jupyter_text>V. Fasttext<jupyter_code>from gensim.models import FastText

fasttext_model = FastText(vector_size=4, window=3, min_count=1, sentences=sentWords, epochs=10)

fasttext_model
fasttext_model.wv['karim']
fasttext_model.wv['document']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10538800406641817, 'fraction_numerical': 0.014571331751948491, 'mean_word_length': 6.417085427135678, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '383957', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1085, 'n_words': 272, 'path': 'tuto/CH06/encoding_python_gensim.ipynb', 'repo_name': 'projeduc/ESI_2CS_TALN', 'size': '2951'}