<jupyter_start><jupyter_text>Пример работы с TopicPriorRegularizerВ данном ноутбуке рассматривается применение и влияние на результат TopicPriorRegularizer и его версии с семплированием априорных вероятностей $\varphi_t$ из распределения Дирихле.  TopicPriorRegularizer$$R(\Phi,\Theta) = \sum_t\sum_w\beta_t\ln\phi_{wt}$$$$\frac{\partial R}{\partial\Phi_{wt}} = \frac{\beta_t}{\varphi_{wt}}$$<jupyter_code>import numpy as np
import pandas as pd
from numpy import random
from matplotlib import pyplot as plt

import artm
import sys
sys.path.append('../../')

from topicnet.cooking_machine.dataset import Dataset

from topicnet.cooking_machine.models import TopicModel
from topicnet.cooking_machine.models.semantic_radius_score import SemanticRadiusScore
from topicnet.cooking_machine.models.topic_prior_regularizer import TopicPriorRegularizer
from topicnet.cooking_machine.models.topic_prior_regularizer import TopicPriorSampledRegularizer


from topicnet.viewers.top_tokens_viewer import TopTokensViewer
from topicnet.viewers.top_documents_viewer import TopDocumentsViewer<jupyter_output><empty_output><jupyter_text>Загрузка данных Используемая для моделирования коллекция содержит 1000 документов по теме "Бизнес" и 100 документов по теме "Музыка".<jupyter_code>DATA_PATH = 'data/lenta_1000_100.csv'

dataset_lenta = Dataset(DATA_PATH, batch_size=10)
dictionary_lenta = dataset_lenta.get_dictionary()
batch_vectorizer_lenta = dataset_lenta.get_batch_vectorizer()
pd.read_csv(DATA_PATH).groupby('tags').count()['id']<jupyter_output><empty_output><jupyter_text>Проведение обучения на 2 темы<jupyter_code>pd.read_csv('data/lenta_1000_100.csv').groupby('tags').count()
topic_prior_reg = TopicPriorRegularizer(name='topic_prior', tau=5, num_topics=2, beta=[10, 1])
model_lenta = artm.ARTM(num_topics=2, cache_theta=True)
model_lenta.scores.add(artm.scores.PerplexityScore(name='perplexity'))
model_lenta.initialize(dictionary_lenta)

model_lenta_unbalanced = artm.ARTM(num_topics=2, cache_theta=True)
model_lenta_unbalanced.scores.add(artm.scores.PerplexityScore(name='perplexity'))
model_lenta_unbalanced.initialize(dictionary_lenta)
topic_model_lenta = TopicModel(artm_model=model_lenta, model_id='model_lenta')
topic_model_lenta_unbalanced = TopicModel(artm_model=model_lenta_unbalanced, model_id='model_lenta_unbalanced')
topic_model_lenta._fit(batch_vectorizer_lenta, 10, custom_regularizers={topic_prior_reg.name: topic_prior_reg})

topic_model_lenta_unbalanced._fit(batch_vectorizer_lenta, 10)
print('Баланс документов в модели с регуляризатором:\n{}\n'.format(
    topic_model_lenta.get_theta().idxmax().value_counts()
))

print('Баланс документов в модели без регуляризатора:\n{}'.format(
    topic_model_lenta_unbalanced.get_theta().idxmax().value_counts()
))
plt.figure(figsize=(14,4))

plt.subplot(1,2,1)
values = np.array(model_lenta.score_tracker['perplexity'].value)
med = np.median(values)
values[values > 5 * med] = np.nan

plt.title('Перплексия для модели с регуляризатором')
plt.plot(values)

plt.subplot(1,2,2)
values = np.array(model_lenta_unbalanced.score_tracker['perplexity'].value)
med = np.median(values)
values[values > 5 * med] = np.nan

plt.title('Перплексия для модели без регуляризатора')
plt.plot(values)
tokens_viewer = TopTokensViewer(model=topic_model_lenta, num_top_tokens=10)
top_tokens_dict = tokens_viewer.view()
    
for topic_id, topic in enumerate(top_tokens_dict.values()):
    print('Topic {}:\n\nTop ngramms: {}\n\nTop words: {}\n\n'.format(
        str(topic_id), ', '.join(topic['ngramms'].keys()), ', '.join(topic['text'].keys())
         ))
tokens_viewer_unbalanced = TopTokensViewer(model=topic_model_lenta_unbalanced, num_top_tokens=10)
top_tokens_dict_unbalanced = tokens_viewer_unbalanced.view()
    
for topic_id, topic in enumerate(top_tokens_dict_unbalanced.values()):
    print('Topic {}:\n\nTop ngramms: {}\n\nTop words: {}\n\n'.format(
        str(topic_id), ', '.join(topic['ngramms'].keys()), ', '.join(topic['text'].keys())
         ))
# Вычисление радиусов семантической однородности требует времени

score = SemanticRadiusScore(batch_vectorizer=batch_vectorizer_lenta)
%%time

score.call(topic_model_lenta)
%%time

score.call(topic_model_lenta_unbalanced)<jupyter_output>CPU times: user 2min 50s, sys: 44.2 s, total: 3min 34s
Wall time: 2min 11s
{'fraction_non_alphanumeric': 0.11890385508592662, 'fraction_numerical': 0.014398513701811427, 'mean_word_length': 7.917184265010352, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '605885', 'license': 'MIT', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1637, 'n_words': 245, 'path': 'topicnet/demos/topic_prior_regularizer_tutorial.ipynb', 'repo_name': 'tkhirianov/TopicNet', 'size': '4306'}