<jupyter_start><jupyter_text>Classes correspondence in AG News* 1 - World* 2 - Sports* 3 - Business* 4 - Science and technology<jupyter_code>def get_tokenized_data(dataset):
    return [example.text for example in dataset.examples]

tokenized_data_train = get_tokenized_data(train)
tokenized_data_test = get_tokenized_data(test)
def get_labels(dataset):
    return np.array( [example.label for example in dataset.examples] )
labels_test = get_labels(test)
num_topics = len(label_field.vocab)

# Build a Dictionary - association word to numeric id
dictionary = corpora.Dictionary(tokenized_data_train)
 
# Transform the collection of texts to a numerical form
corpus = [dictionary.doc2bow(text) for text in tokenized_data_train]

# Build the LDA model
lda_model = models.LdaModel(corpus=corpus, num_topics=num_topics, id2word=dictionary)
corpus_test = [dictionary.doc2bow(text) for text in tokenized_data_test]
# Get topics probabilities
def get_topics_probabilities(lda_model):
    topic_dist_test = lda_model[corpus_test]
    results = []
    for lda_topic in range(num_topics):
         for real_topic in label_field.vocab.stoi.keys():
                y_true = (labels_test == real_topic)
                support = sum(y_true)
                y_score = [dict(probs)[lda_topic] if lda_topic in dict(probs).keys() else 0. 
                           for probs in np.array(topic_dist_test) ]
                pr_auc = average_precision_score(y_true=y_true, y_score=y_score)          
                results.append([lda_topic, real_topic, pr_auc, support])

    results_df = pd.DataFrame(results, columns=['lda_topic','real_topic','pr_auc','support']).sort_values('pr_auc', ascending=False)
    return results_df

results_df = get_topics_probabilities(lda_model)
def get_best_model_and_topics(results_df):
    classes = dict(label_field.vocab.stoi)
    model_perf_data = []
    for row in results_df.values:
        class_ = row[1]
        if class_ in classes.keys():
            lda_topic = row[0]
            pr_auc = row[2]
            support = row[3]
            model_perf_data.append([class_, lda_topic, pr_auc, support])
            classes.pop(class_)
    model_perf_df = pd.DataFrame( model_perf_data )
    mean_perf = model_perf_df[2].mean()
    return model_perf_df, mean_perf
    
model_perf_df, mean_perf = get_best_model_and_topics(results_df)
print(model_perf_df)
print('Mean performance:', mean_perf)
alphas = [0.1, 0.25, 0.5, 0.75, 1, 1.25 ]
etas = [0.1, 0.25, 0.5, 0.75, 1, 1.25 ]
model_perf_df_results = {}
mean_perf_results = {}
all_models = {}

best_perf = 0

for alpha in tqdm(alphas):
    for eta in etas:
        lda_model = models.LdaModel(corpus=corpus, num_topics=num_topics, id2word=dictionary, passes=5, distributed=False, alpha=alpha, eta=eta)
        results_df = get_topics_probabilities(lda_model)
        model_perf_df, mean_perf = get_best_model_and_topics(results_df)
        model_perf_df_results[str([alpha, eta])] = model_perf_df
        mean_perf_results[str([alpha, eta])] = mean_perf
        all_models[str([alpha, eta])] = lda_model
        
        if mean_perf > best_perf: 
            best_perf = mean_perf
            #lda_model.save('best_model.pickle')
            print('->',alpha, eta, best_perf)
        else:
            print(alpha, eta, mean_perf)
        
        
        pickle.dump( model_perf_df_results, open('model_perf_df_results_'+DATASET+'_'+VERSION+'.pickle', 'wb'))
        pickle.dump( mean_perf_results, open('mean_perf_results_'+DATASET+'_'+VERSION+'.pickle', 'wb'))
        pickle.dump( all_models, open('all_models_'+DATASET+'_'+VERSION+'.pickle', 'wb'))<jupyter_output>0%|          | 0/6 [00:00<?, ?it/s]
{'fraction_non_alphanumeric': 0.11092985318107668, 'fraction_numerical': 0.013866231647634585, 'mean_word_length': 3.575870646766169, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '618009', 'license': 'MIT', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1269, 'n_words': 244, 'path': 'method.ipynb', 'repo_name': 'manuelsh/topic-identification-and-evaluation', 'size': '3678'}