<jupyter_start><jupyter_text>Word2Vec<jupyter_code>#corpus na podstawie tytulu
corpus = df['title'].map(simple_preprocess)
#lista corpusu
corpus
#model na podstawie corpusu
model = Word2Vec(corpus, size = 100, window = 5 , min_count = 1)
model.wv.most_similar('deep')<jupyter_output><empty_output><jupyter_text>Przykład<jupyter_code>#dzialanie Phrases
#polaczenie 2 slow kolo sie (machine learning ==> machine_learning)
corpus = [
    ['a','b','c'],
    ['x','x','w','a','b','q'],
    ['d','a','b', 'v','k','e'],
    ['z','q','a','b']

]

bigram = Phrases(corpus, min_count = 1, threshold = 1)
#działa podobnie Phraser - glównie do opytmalizacji
#bigram = Phraser(Phrases(corpus, min_count = 1, threshold = 1))

bigram  [['x','d','a','b','e']]<jupyter_output><empty_output><jupyter_text>Title + Phrases<jupyter_code>title_corpus = df['title'].map(simple_preprocess)

title_bigram = Phrases(title_corpus, min_count = 1, threshold = 1)

#pojeduncze slowa
simple_preprocess('Machine Learning / Data Scientist')
#sklejone slowa na podstawie wyuczonego title_bigram Phrases
title_bigram [simple_preprocess('Machine Learning / Data Scientist')]
#dla calego zbioru danych przpuszczamy przez Phrase
title_corpus_phrase = [title_bigram[sent] for sent in title_corpus]
model = Word2Vec(title_corpus_phrase, size = 100, window = 5 , min_count = 1)
model.wv.most_similar('database')
#przykald
#rozdzielone slowa
simple_preprocess('Machine Learning / Data Scientist')
#skleojone slowa
title_bigram [simple_preprocess('Machine Learning / Data Scientist')]
# zbior sklejone + osobne slowa :-)
['machine', 'learning', 'data', 'scientist'] + ['machine_learning', 'data_scientist']
#bigram[sent] + sent - zbior sklejone + osobne slowa  
def prepare_corpus(corpus, bigram):
    return [bigram[sent] + sent for sent in corpus]
def prepare_corpus(corpus, bigram):
    for sent in corpus:
        yield bigram[sent] + sent
ext_corp = list( prepare_corpus(title_corpus, title_bigram))
title_model = Word2Vec(ext_corp, size = 100, window = 2, min_count = 1)
title_model = Word2Vec(ext_corp, size = 100, window = 2, min_count = 1)
title_model.wv.most_similar('oracle')<jupyter_output><empty_output><jupyter_text>Description<jupyter_code>df.sample()['description'].values
simple_preprocess(df.sample()['description'].values[0])
#corpus i bigram dla description
description_corpus = df['description'].map(simple_preprocess)
description_bigram = Phrases(description_corpus, min_count = 1, threshold = 1)
#zbior sklejone + osobne slowa  
ext_description_corp = list( prepare_corpus(description_corpus, description_bigram))
#model na czystych danych
description_model_corpus = Word2Vec(description_corpus, size = 100, window = 2, min_count = 1)
description_model_corpus.wv.most_similar('etl')
##model na czystych danych + sklejone
description_model_ext_corp = Word2Vec(ext_description_corp, size = 100, window = 2, min_count = 1)
description_model_ext_corp.wv.most_similar('etl')<jupyter_output><empty_output><jupyter_text>Wizualizacja<jupyter_code>#wymieszanie zdan ze zbioru wejsciowego aby nie bylo zaleznosci
sentences_light = np.random.permutation (title_corpus)

#tworzy slownik z unikalnymi slowami
dictionary = Dictionary (sentences_light)
#zlicza ilosc slow w zdaniu
%time bow_corpus = [dictionary.doc2bow(sent) for sent in sentences_light ]
# LDA 
# bow_corpus
# dictionary
# num_topics - ilosc tematow
# passes - ilosc iteracji
%time lda_model = LdaMulticore(bow_corpus, id2word= dictionary, num_topics = 100, passes = 20)
lda_vis = pyLDAvis.gensim.prepare(lda_model,bow_corpus,dictionary)
pyLDAvis.display(lda_vis)<jupyter_output>C:\Users\tskrzypczak\AppData\Local\Continuum\anaconda3\lib\site-packages\pyLDAvis\_prepare.py:257: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  return pd.concat([default_term_info] + list(topic_dfs))
{'fraction_non_alphanumeric': 0.11966236345580933, 'fraction_numerical': 0.014399205561072492, 'mean_word_length': 7.090361445783133, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '232255', 'license': 'MIT', 'n_tokens_mistral': 1520, 'n_tokens_neox': 1468, 'n_words': 318, 'path': 'part4/day4/day4.ipynb', 'repo_name': 'toskpl/Challenge_DW', 'size': '4028'}