<commit_before>import os

from gensim import corpora, models

path = 'resources'
parent_name = 'Honda'
corpus_path = path + '/' + parent_name + '.mm'
dictionary_path = path + '/' + parent_name + '.dict'
model_path = path + '/' + parent_name + '.model'

corpus = corpora.MmCorpus(corpus_path)
dictionary = corpora.Dictionary.load(dictionary_path)

hdp = models.HdpModel(corpus, id2word=dictionary)


def save_model(model):
    if not os.path.isfile(model_path):
        with open(model_path, 'wb') as fh:
            model.save(fh)


save_model(hdp)


def load_model():
    return models.HdpModel.load(model_path)

hdp = load_model()

topics = hdp.print_topics(num_words=12)
for topic in topics:
    print(topic)<commit_msg>prettify data output - json created<commit_after>import csv
import os

import re
from gensim import corpora, models

path = 'resources'
parent_name = 'Honda'
corpus_path = path + '/' + parent_name + '.mm'
dictionary_path = path + '/' + parent_name + '.dict'
model_path = path + '/' + parent_name + '.model'

corpus = corpora.MmCorpus(corpus_path)
dictionary = corpora.Dictionary.load(dictionary_path)

hdp = models.HdpModel(corpus, id2word=dictionary)


def save_model(model):
    if not os.path.isfile(model_path):
        with open(model_path, 'wb') as fh:
            model.save(fh)


save_model(hdp)


def load_model():
    return models.HdpModel.load(model_path)

hdp = load_model()

topics = hdp.print_topics(num_words=12)


def pretty_output(ts):

    def save_to_csv(ts):
        out = list()
        regex = re.compile('[^a-zA-Z]')
        for t in ts:
            row = list()
            row.append('Topic ' +str(t[0]+1))
            li = t[1].split('+')
            li = [x.strip() for x in li]
            li = [regex.sub('', x) for x in li]
            row.extend(li)
            out.append(row)
            # print(row) #  DEBUG
        f = open('resources/data.csv', 'w')
        w = csv.writer(f)
        w.writerows(out)
        f.close()

    save_to_csv(ts)

    def create_json(ts):
        out = dict()
        regex = re.compile('[^a-zA-Z]')
        for t in ts:
            key = 'Topic ' + str(t[0] + 1)
            li = t[1].split('+')
            li = [x.strip() for x in li]
            li = [regex.sub('', x) for x in li]
            out[key] = li
        return out

    print(create_json(ts))


pretty_output(topics)
{'fraction_non_alphanumeric': 0.14050632911392405, 'fraction_numerical': 0.005063291139240506, 'mean_word_length': 2.581570996978852, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3469498', 'n_tokens_mistral': 874, 'n_tokens_neox': 836, 'n_words': 190}