<jupyter_start><jupyter_text>Metadata```Course:   DS 5001Module:   00 Final ProjectsTopic:    Using MALLET```  NotesSee Appendix below information on how download and install MALLET.  Set Up<jupyter_code>data_home = "../data"
local_lib = "../lib"
data_prefix = 'novels'
OHCO = ['book_id','chap_id','para_num','sent_num','token_num']
max_words = 10000

# For MALLET
num_topics = 20
num_iters = 1000
show_interval = 100
import pandas as pd
import numpy as np<jupyter_output><empty_output><jupyter_text>Import CORPUS<jupyter_code>LIB = pd.read_csv(f"{data_home}/{data_prefix}/{data_prefix}-LIB.csv").set_index(OHCO[:1])
CORPUS = pd.read_csv(f"{data_home}/{data_prefix}/{data_prefix}-CORPUS.csv").set_index(OHCO)
CORPUS.head()<jupyter_output><empty_output><jupyter_text>Create DOC<jupyter_code>def gather_docs(CORPUS, ohco_level, term_col='term_str'):
    OHCO = CORPUS.index.names
    CORPUS[term_col] = CORPUS[term_col].astype('str')
    DOC = CORPUS.groupby(OHCO[:ohco_level])[term_col].apply(lambda x:' '.join(x)).to_frame('doc_str')
    return DOC
DOC = gather_docs(CORPUS, 2)
DOC['n_tokens'] = DOC.doc_str.apply(lambda x: len(x.split()))
DOC<jupyter_output><empty_output><jupyter_text>Dump corpus to CSV file<jupyter_code>mallet_corpus = DOC.join(LIB)[['doc_str','author_id']]
mallet_corpus.columns = 'doc_content doc_label'.split()
mallet_corpus[['doc_label','doc_content']].to_csv('novels-corpus.csv', index=False)<jupyter_output><empty_output><jupyter_text>MALLET Time  Show MALLET options<jupyter_code>mallet_home = "/Users/rca2t1/opt/mallet/bin"
! which mallet<jupyter_output>mallet not found<jupyter_text>Import corpus<jupyter_code>!{mallet_home}/mallet import-file --input novels-corpus.csv --output novels-corpus.mallet --keep-sequence TRUE<jupyter_output><empty_output><jupyter_text>Train topics<jupyter_code>!{mallet_home}/mallet train-topics --input novels-corpus.mallet --num-topics {num_topics} --num-iterations {num_iters} \
--output-doc-topics novels-doc-topics.txt \
--output-topic-keys novels-topic-keys.txt \
--word-topic-counts-file novels-word-topic-counts-file.txt \
--topic-word-weights-file novels-topic-word-weights-file.txt \
--xml-topic-report novels-topic-report.xml \
--xml-topic-phrase-report novels-topic-phrase-report.xml \
--show-topics-interval {show_interval} \
--use-symmetric-alpha false  \
--optimize-interval 100 \
--diagnostics-file novels-diagnostics.xml<jupyter_output>Mallet LDA: 20 topics, 5 topic bits, 11111 topic mask
Data loaded.
max tokens: 15717
total tokens: 1164070
<10> LL/token: <PASSWORD>
<20> LL/token: <PASSWORD>
<30> LL/token: <PASSWORD>
<40> LL/token: <PASSWORD>
<50> LL/token: <PASSWORD>
<60> LL/token: <PASSWORD>
<70> LL/token: <PASSWORD>
<80> LL/token: <PASSWORD>
<90> LL/token: <PASSWORD>

0	0.25	the emily said and not but montoni was that from for she annette had madame this what castle who heard 
1	0.25	and was had that which but the with when his could for from heart not yet then before into all 
2	0.25	the that have and been was not but this were upon one which from would for its some will what 
3	0.25	the with which not this their are than our more his must has may and its who for into full 
4	0.25	and the that for not all her his she with when there but then our some shall van was him 
5	0.25	the his was upon from with not and your him but for every were matilda agnes antonia what have ambrosio 
6	0.25	the that was you tuppence tommy she his sir p[...]
{'fraction_non_alphanumeric': 0.12809315866084425, 'fraction_numerical': 0.027074235807860263, 'mean_word_length': 5.339483394833948, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '150579', 'license': 'MIT', 'n_tokens_mistral': 1347, 'n_tokens_neox': 1255, 'n_words': 315, 'path': 'M00_FinalProjects/M00_04_MALLET.ipynb', 'repo_name': 'avnimalik/DS5001-2022-01', 'size': '3417'}