<jupyter_start><jupyter_text>CFPB Student Loans complaints HDP https://www.consumerfinance.gov/data-research/consumer-complaints/download-the-data<jupyter_code>import numpy as np
import spacy
nlp = spacy.load('en_core_web_sm')
file_student='../data/consumercomplaints/student_comp_narrative.txt'
f=open(file_student,'r',encoding='utf-8')
student_text=f.read()
f.close()
doc_student=nlp(student_text)
student_pos_list=['NOUN']
student_preproc_text=[]
student_preproc_sent=[]

for token in doc_student:
    if token.text!='\n':
        if not(token.is_stop) and not(token.is_punct) and token.pos_ in student_pos_list:
            student_preproc_sent.append(token.lemma_)
    else:
        student_preproc_text.append(student_preproc_sent)
        student_preproc_sent=[]

student_preproc_text.append(student_preproc_sent) #last sentence

print(student_preproc_text)
import tomotopy as tp
mdl = tp.HDPModel(alpha=0.1,seed=0)

for line in student_preproc_text:
    mdl.add_doc(line)

mdl.train(50)
print('Log Perplexity=', mdl.ll_per_word)

for k in range(mdl.k):
    print('Top 10 words of topic #{}'.format(k))
    print(mdl.get_topic_words(k, top_n=10))

mdl.save('../data/consumercomplaints/hdp_model.bin')
mdl = tp.HDPModel.load('../data/consumercomplaints/hdp_model.bin')
bag_of_words=[word for sent in student_preproc_text for word in sent]
doc_inst = mdl.make_doc(bag_of_words)
np.argsort(np.array(mdl.infer(doc_inst)[0]))[::-1]
print(mdl.get_topic_words(5, top_n=7))
print(mdl.get_topic_words(7, top_n=7))
print(mdl.get_topic_words(4, top_n=7))<jupyter_output>[('employer', 0.03343059867620468), ('graduation', 0.03343059867620468), ('book', 0.03343059867620468), ('diploma', 0.025093790143728256), ('debt', 0.025093790143728256), ('education', 0.025093790143728256), ('college', 0.025093790143728256)]<jupyter_text>CFPB Student Loans complaints LDA<jupyter_code>NUM_TOPICS=20

mdl = tp.LDAModel(k=NUM_TOPICS,alpha=0.1,seed=0)

for line in student_preproc_text:
    mdl.add_doc(line)

mdl.train(50)
print('Log Perplexity=', mdl.ll_per_word)

for k in range(mdl.k):
    print('Top 10 words of topic #{}'.format(k))
    print(mdl.get_topic_words(k, top_n=10))

mdl.save('../data/consumercomplaints/lda_model.bin')
mdl = tp.LDAModel.load('../data/consumercomplaints/lda_model.bin')
bag_of_words=[word for sent in student_preproc_text for word in sent]
doc_inst = mdl.make_doc(bag_of_words)
np.argsort(np.array(mdl.infer(doc_inst)[0]))[::-1]
print(mdl.get_topic_words(17, top_n=7))
print(mdl.get_topic_words(7, top_n=7))
print(mdl.get_topic_words(6, top_n=7))<jupyter_output>[('loan', 0.24387744069099426), ('time', 0.06379450112581253), ('student', 0.051527272909879684), ('m', 0.05103658139705658), ('money', 0.04514831304550171), ('payment', 0.03239039331674576), ('collection', 0.027974190190434456)]
{'fraction_non_alphanumeric': 0.15359942979330007, 'fraction_numerical': 0.10548823948681398, 'mean_word_length': 6.840782122905028, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '270568', 'license': 'MIT', 'n_tokens_mistral': 1332, 'n_tokens_neox': 1146, 'n_words': 165, 'path': 'Chapter05/Activity 5.02/Activity 5.02.ipynb', 'repo_name': 'abhishekr128/The-Natural-Language-Processing-Workshop', 'size': '2806'}