<jupyter_start><jupyter_text>Load data<jupyter_code># Load pre-saved BoW
# Save BoW
user_dir = os.path.expanduser('~/cltk_data/user_data/')
try:
    os.makedirs(user_dir)
except FileExistsError:
    pass
bow_path = os.path.join(user_dir, 'bow_lda_gensim.mm')

mm_corpus = gensim.corpora.MmCorpus(bow_path)
print(mm_corpus)
print(next(iter(mm_corpus)))<jupyter_output>[(0, 1.0), (1, 1.0), (2, 1.0), (3, 1.0), (4, 1.0), (5, 1.0), (6, 1.0), (7, 2.0), (8, 1.0), (9, 1.0), (10, 1.0), (11, 1.0), (12, 1.0), (13, 1.0), (14, 1.0), (15, 1.0), (16, 1.0), (17, 1.0), (18, 1.0), (19, 1.0), (20, 1.0), (21, 1.0), (22, 1.0), (23, 1.0), (24, 1.0), (25, 1.0), (26, 1.0), (27, 1.0), (28, 1.0), (29, 1.0), (30, 1.0), (31, 1.0), (32, 1.0), (33, 1.0), (34, 1.0), (35, 1.0), (36, 1.0), (37, 1.0), (38, 1.0), (39, 1.0), (40, 1.0), (41, 1.0), (42, 1.0), (43, 1.0), (44, 1.0), (45, 1.0), (46, 1.0), (47, 1.0), (48, 2.0), (49, 1.0), (50, 1.0), (51, 1.0), (52, 1.0), (53, 1.0), (54, 1.0), (55, 1.0), (56, 1.0), (57, 1.0), (58, 1.0), (59, 1.0), (60, 1.0), (61, 1.0), (62, 1.0), (63, 1.0), (64, 1.0), (65, 1.0), (66, 1.0), (67, 1.0), (68, 1.0), (69, 1.0)]<jupyter_text>Semantic transformationshttp://radimrehurek.com/topic_modeling_tutorial/2%20-%20Topic%20Modeling.htmlSemantic-transformations  LDA<jupyter_code># Save for reuse
with open(os.path.expanduser('~/cltk_data/user_data/tlg_bow_id2word.dict'), 'rb') as file_open:
    id2word_tlg = pickle.load(file_open)
# Quick testing using just a part of the corpus

# use fewer documents during training, LDA is slow
clipped_corpus = gensim.utils.ClippedCorpus(mm_corpus, 100)
%time lda_model = gensim.models.LdaModel(clipped_corpus,
                                         num_topics=10,
                                         id2word=id2word_tlg,
                                         passes=4)
lda_model.print_topics(-1)  # print a few most important words for each LDA topic<jupyter_output><empty_output><jupyter_text>TI-IDF + LSI<jupyter_code># first train tfidf model
# this modifies the feature weights of each word
%time tfidf_model = gensim.models.TfidfModel(mm_corpus, id2word=id2word_tlg)
# then run lsi, which reduces dimensionality
%time lsi_model = gensim.models.LsiModel(tfidf_model[mm_corpus], id2word=id2word_tlg, num_topics=200)
# for the first doc of the TLG corpus, here are the LSI scores for each of the 200 topics
print(next(iter(lsi_model[tfidf_model[mm_corpus]])))<jupyter_output>[(0, 0.024289037065511311), (1, 0.014077893150839215), (2, -0.019146078135534279), (3, -0.018767966018989272), (4, -0.00099884333481120103), (5, -0.0051913135174345487), (6, -0.0014277098382469583), (7, -0.018818824919966906), (8, 0.0056374952350573453), (9, -0.0023221873083177956), (10, -0.0081668241786406002), (11, -0.019996384848007791), (12, 0.014338300486609062), (13, -0.00010303379864312642), (14, 0.0021088994788161335), (15, 0.0098757069667030732), (16, 0.0090732013832107734), (17, 0.0081570602885538118), (18, -0.0098302970910077361), (19, 0.01134303047809858), (20, -0.0042868477183203814), (21, -0.0019021128579354079), (22, 0.0083599609888375184), (23, -0.0024645892529614947), (24, -0.0089570567476525975), (25, 0.013241931401700954), (26, -0.0079806310637484165), (27, 0.0037053228065251193), (28, 0.002579960226988357), (29, 0.016675254728906144), (30, 0.001139717059664551), (31, 0.017415051948121849), (32, 0.02155588300179399), (33, -0.014399643194274744), (34, 0.00864178216383[...]<jupyter_text>Save corpora<jupyter_code># cache the transformed corpora to disk, for use in later notebooks
path_lda = os.path.join(user_dir, 'gensim_tlg_lda.mm')
path_tfidf = os.path.join(user_dir, 'gensim_tlg_tfidf.mm')
path_lsi= os.path.join(user_dir, 'gensim_tlg_lsa.mm')
%time gensim.corpora.MmCorpus.serialize(path_lda, lda_model[mm_corpus])
%time gensim.corpora.MmCorpus.serialize(path_tfidf, tfidf_model[mm_corpus])
%time gensim.corpora.MmCorpus.serialize(path_lsi, lsi_model[tfidf_model[mm_corpus]])<jupyter_output>INFO : storing corpus in Matrix Market format to /home/kyle/cltk_data/user_data/gensim_tlg_lda.mm
INFO : saving sparse matrix to /home/kyle/cltk_data/user_data/gensim_tlg_lda.mm
INFO : PROGRESS: saving document #0
INFO : PROGRESS: saving document #1000
INFO : saved 1484x10 matrix, density=56.617% (8402/14840)
INFO : saving MmCorpus index to /home/kyle/cltk_data/user_data/gensim_tlg_lda.mm.index
INFO : storing corpus in Matrix Market format to /home/kyle/cltk_data/user_data/gensim_tlg_tfidf.mm
INFO : saving sparse matrix to /home/kyle/cltk_data/user_data/gensim_tlg_tfidf.mm
INFO : PROGRESS: saving document #0<jupyter_text>Transform "unseen" documentsWhat is the best way to get LDA scores on a corpus used for training a model?<jupyter_code># LDA

def tokenize(text):
    # https://radimrehurek.com/gensim/utils.html#gensim.utils.simple_preprocess
    tokens = [token for token in simple_preprocess(text, deacc=True)]
    return [token for token in tokens if token not in STOPS_LIST]

doc = "ἐπειδὴ πᾶσαν πόλιν ὁρῶμεν κοινωνίαν τινὰ οὖσαν καὶ πᾶσαν κοινωνίαν ἀγαθοῦ τινος ἕνεκεν συνεστηκυῖαν （τοῦ γὰρ εἶναι δοκοῦντος ἀγαθοῦ χάριν πάντα πράττουσι πάντες）, δῆλον ὡς πᾶσαι μὲν ἀγαθοῦ τινος στοχάζονται, μάλιστα δὲ [5] καὶ τοῦ κυριωτάτου πάντων ἡ πασῶν κυριωτάτη καὶ πάσας περιέχουσα τὰς ἄλλας. αὕτη δ᾽ ἐστὶν ἡ καλουμένη πόλις καὶ ἡ κοινωνία ἡ πολιτική. ὅσοι μὲν οὖν οἴονται πολιτικὸν καὶ βασιλικὸν καὶ οἰκονομικὸν καὶ δεσποτικὸν εἶναι τὸν αὐτὸν οὐ καλῶς λέγουσιν （πλήθει γὰρ καὶ ὀλιγότητι νομίζουσι [10] διαφέρειν ἀλλ᾽ οὐκ εἴδει τούτων ἕκαστον, οἷον ἂν μὲν ὀλίγων, δεσπότην, ἂν δὲ πλειόνων, οἰκονόμον, ἂν δ᾽ ἔτι πλειόνων, πολιτικὸν ἢ βασιλικόν, ὡς οὐδὲν διαφέρουσαν μεγάλην οἰκίαν ἢ μικρὰν πόλιν: καὶ πολιτικὸν δὲ καὶ βασιλικόν, ὅταν μὲν αὐτὸς ἐφεστήκῃ, βασιλικόν, ὅταν [15] δὲ κατὰ τοὺς λόγους τῆς ἐπιστήμης τῆς τοιαύτης κατὰ μέρος ἄρχων καὶ ἀρχόμενος, πολιτικόν: ταῦτα δ᾽ οὐκ ἔστιν ἀληθῆ）: δῆλον δ᾽ ἔσται τὸ λεγόμενον ἐπισκοποῦσι κατὰ τὴν ὑφηγημένην μέθοδον. ὥσπερ γὰρ ἐν τοῖς ἄλλοις τὸ σύνθετον μέχρι τῶν ἀσυνθέτων ἀνάγκη διαιρεῖν （ταῦτα γὰρ ἐλάχιστα [20] μόρια τοῦ παντός）, οὕτω καὶ πόλιν ἐξ ὧν σύγκειται σκοποῦντες ὀψόμεθα καὶ περὶ τούτων μᾶλλον, τί τε διαφέρουσιν ἀλλήλων καὶ εἴ τι τεχνικὸν ἐνδέχεται λαβεῖν περὶ ἕκαστον τῶν ῥηθέντων."
doc = ' '.join(simple_preprocess(doc))
# transform text into the bag-of-words space
bow_vector = id2word_tlg.doc2bow(tokenize(doc))
print([(id2word_tlg[id], count) for id, count in bow_vector])
# transform into LDA space
lda_vector = lda_model[bow_vector]
print(lda_vector)
# print the document's single most prominent LDA topic
print(lda_model.print_topic(max(lda_vector, key=lambda item: item[1])[0]))
# transform into LSI space
lsi_vector = lsi_model[tfidf_model[bow_vector]]
print(lsi_vector)
# print the document's single most prominent LSI topic (not interpretable like LDA!)
print(lsi_model.print_topic(max(lsi_vector, key=lambda item: abs(item[1]))[0]))<jupyter_output>0.140*"περιαυχενιον" + -0.082*"λεγοιεν" + 0.072*"απονοηθεντων" + -0.065*"κριθαι" + -0.064*"κλαπεντας" + 0.063*"ζωνας" + -0.062*"περιπαθης" + -0.061*"αμμωνος" + 0.061*"λυσικλει" + 0.061*"λαμβανοντα"<jupyter_text>Save models<jupyter_code>path_lda = os.path.join(user_dir, 'gensim_tlg_lda.model')
path_tfidf = os.path.join(user_dir, 'gensim_tlg_tfidf.model')
path_lsi= os.path.join(user_dir, 'gensim_tlg_lsa.model')

# store all trained models to disk
lda_model.save(path_lda)
lsi_model.save(path_lsi)
tfidf_model.save(path_tfidf)<jupyter_output>INFO : saving LdaState object under /home/kyle/cltk_data/user_data/gensim_tlg_lda.model.state, separately None
INFO : storing np array 'sstats' to /home/kyle/cltk_data/user_data/gensim_tlg_lda.model.state.sstats.npy
INFO : saved /home/kyle/cltk_data/user_data/gensim_tlg_lda.model.state
INFO : saving LdaModel object under /home/kyle/cltk_data/user_data/gensim_tlg_lda.model, separately ['expElogbeta', 'sstats']
INFO : storing np array 'expElogbeta' to /home/kyle/cltk_data/user_data/gensim_tlg_lda.model.expElogbeta.npy
INFO : not storing attribute dispatcher
INFO : not storing attribute id2word
INFO : not storing attribute state
INFO : saved /home/kyle/cltk_data/user_data/gensim_tlg_lda.model
INFO : saving Projection object under /home/kyle/cltk_data/user_data/gensim_tlg_lsa.model.projection, separately None
INFO : storing np array 'u' to /home/kyle/cltk_data/user_data/gensim_tlg_lsa.model.projection.u.npy
INFO : saved /home/kyle/cltk_data/user_data/gensim_tlg_lsa.model.projection
INFO : [...]<jupyter_text>EvaluationRead insighful intro at http://radimrehurek.com/topic_modeling_tutorial/2%20-%20Topic%20Modeling.htmlEvaluation  Word intrusion> For each trained topic, they take its first ten words, then substitute one of them with another, randomly chosen word (intruder!) and see whether a human can reliably tell which one it was. If so, the trained topic is topically coherent (good); if not, the topic has no discernible theme (bad)<jupyter_code># select top 50 words for each of the 20 LDA topics
top_words = [[word for word, _ in lda_model.show_topic(topicno, topn=50)] for topicno in range(lda_model.num_topics)]
print(top_words)
# get all top 50 words in all 20 topics, as one large set
all_words = set(itertools.chain.from_iterable(top_words))

print("Can you spot the misplaced word in each topic?")

# for each topic, replace a word at a different index, to make it more interesting
replace_index = np.random.randint(0, 10, lda_model.num_topics)

replacements = []
for topicno, words in enumerate(top_words):
    other_words = all_words.difference(words)
    replacement = np.random.choice(list(other_words))
    replacements.append((words[replace_index[topicno]], replacement))
    words[replace_index[topicno]] = replacement
    print("%i: %s" % (topicno, ' '.join(words[:10])))
print("Actual replacements were:")
print(list(enumerate(replacements)))<jupyter_output>Actual replacements were:
[(0, ('διαφωνει', 'προκομισαντας')), (1, ('αιστωτηριον', 'διακεκαυμενη')), (2, ('κινδυνευοντι', 'νωχος')), (3, ('νεα', 'παθεσιν')), (4, ('επωνομαζετο', 'μανθανοντας')), (5, ('νοτιωτερα', 'διαφωνει')), (6, ('πετευρον', 'ελληνικη')), (7, ('κραυαλλιδαι', 'αναστησας')), (8, ('αγομενης', 'ακρατον')), (9, ('καιροι', 'πυλωροι'))]<jupyter_text>Split doc> We'll split each document into two parts, and check that 1) topics of the first half are similar to topics of the second 2) halves of different documents are mostly dissimilar<jupyter_code># this function first defined in pt. 1
def iter_tlg(tlg_dir):
    file_names = os.listdir(tlg_dir)
    for file_name in file_names:
        file_path = os.path.join(tlg_dir, file_name)
        with open(file_path) as file_open:
            file_read = file_open.read()
        tokens = tokenize(file_read)
        # ignore short docs
        if len(tokens) < 50:
            continue
        yield file_name, tokens

# evaluate on 1k documents **not** used in LDA training
tlg_preprocessed = os.path.expanduser('~/cltk_data/greek/text/tlg/plaintext/')
doc_stream = (tokens for _, tokens in iter_tlg(tlg_preprocessed))  # generator
test_docs = list(itertools.islice(doc_stream, 100, 200))  # ['πανυ', 'καλως', ...], [...], ...]
def intra_inter(model, test_docs, num_pairs=10000):
    # split each test document into two halves and compute topics for each half
    part1 = [model[id2word_tlg.doc2bow(tokens[: len(tokens) // 2])] for tokens in test_docs]
    part2 = [model[id2word_tlg.doc2bow(tokens[len(tokens) // 2 :])] for tokens in test_docs]
    
    # print computed similarities (uses cossim)
    print("average cosine similarity between corresponding parts (higher is better):")
    print(np.mean([gensim.matutils.cossim(p1, p2) for p1, p2 in zip(part1, part2)]))

    random_pairs = np.random.randint(0, len(test_docs), size=(num_pairs, 2))
    print("average cosine similarity between {} random parts (lower is better):".format(num_pairs))    
    print(np.mean([gensim.matutils.cossim(part1[i[0]], part2[i[1]]) for i in random_pairs]))
print("LDA results:")
intra_inter(lda_model, test_docs)
print("LSI results:")
intra_inter(lsi_model, test_docs)<jupyter_output>LSI results:
average cosine similarity between corresponding parts (higher is better):
0.759159191756
average cosine similarity between 10000 random parts (lower is better):
0.556656437511<jupyter_text>How to use on corpusJust send convert list of tokens to bow, then send to model: `print(lda_model[id2word_tlg.doc2bow(tokens)])`<jupyter_code>for title, tokens in iter_tlg(tlg_preprocessed):
    #print(title, tokens[:10])  # print the article title and its first ten tokens
    print(title)
    print(lda_model[id2word_tlg.doc2bow(tokens)])
    print('')<jupyter_output>TLG2346.TXT
[(2, 0.77020063255871529), (4, 0.050162662423303819), (7, 0.17759554457537929)]

TLG1389.TXT
[(1, 0.077333330171680217), (2, 0.81018572424497037), (4, 0.029298653619360696), (7, 0.021474921514088502), (9, 0.050474519337949199)]

TLG0404.TXT
[(1, 0.02048927219191567), (2, 0.94907945485028755), (4, 0.029755487376673027)]

TLG0235.TXT
[(2, 0.9770261992926299), (4, 0.015441547453474994)]

TLG0535.TXT
[(1, 0.034813206783092389), (2, 0.71772563652669896), (3, 0.014290175511093661), (4, 0.090466392452987279), (5, 0.0652139733662397), (7, 0.018721921830686009), (9, 0.055341464745294021)]

TLG0507.TXT
[(1, 0.10259625305824292), (2, 0.41373140862637803), (4, 0.14912418250002712), (5, 0.12201859250883705), (7, 0.16176346421143439), (9, 0.050147784476549384)]

TLG1816.TXT
[(1, 0.024000415375272359), (2, 0.86327207953334895), (4, 0.028950050907302946), (5, 0.02272453752338216), (7, 0.035543720007879025), (9, 0.013945883813084248)]

TLG0476.TXT
[(1, 0.038183750715045969), (2, 0.927640218[...]<jupyter_text>VisualizationFollowing: http://nbviewer.jupyter.org/github/bmabey/pyLDAvis/blob/master/notebooks/pyLDAvis_overview.ipynb<jupyter_code>lda_model.show_topics()
import pyLDAvis.gensim

pyLDAvis.enable_notebook()
pyLDAvis.gensim.prepare(lda_model, mm_corpus, id2word_tlg)<jupyter_output>/home/kyle/cltk/venv_pypi/lib/python3.6/site-packages/pyLDAvis/_prepare.py:387: DeprecationWarning: 
.ix is deprecated. Please use
.loc for label based indexing or
.iloc for positional indexing

See the documentation here:
http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix
  topic_term_dists = topic_term_dists.ix[topic_order]
{'fraction_non_alphanumeric': 0.13712444849079067, 'fraction_numerical': 0.13425309895650955, 'mean_word_length': 5.386404293381037, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '398830', 'license': 'MIT', 'n_tokens_mistral': 7851, 'n_tokens_neox': 6173, 'n_words': 1317, 'path': 'lda/leipzig/1_lda_tests/3 LDA basics, pt 2.ipynb', 'repo_name': 'kylepjohnson/ipython_notebooks', 'size': '14279'}