<jupyter_start><jupyter_text>Topic Fingerprints<jupyter_code>import pandas as pd

pd.set_option('display.max_colwidth', 800)
OPEN_DATA_URL = 'data/canada-open-data/inventory.csv'
COLUMNS = ['title_en', 'description_en','date_released']
catalog = pd.read_csv(OPEN_DATA_URL, usecols=COLUMNS)
catalog = catalog.dropna(subset=['description_en'])
catalog
%autosave 60
def text_to_tokens(text):
    text = text.lower()
    text = remove_stopwords(text)
    tokens = simple_preprocess(text)
    return tokens
from gensim.parsing.preprocessing import remove_stopwords
from gensim.utils import simple_preprocess

dataset_descriptions = catalog.description_en.apply(text_to_tokens)
dataset_descriptions
import gensim
from gensim.models import LdaModel
from gensim.parsing.preprocessing import preprocess_string

dictionary = gensim.corpora.Dictionary(dataset_descriptions)
corpus = [dictionary.doc2bow(text) for text in dataset_descriptions]<jupyter_output><empty_output><jupyter_text>Dictionary<jupyter_code>for i in range(0, 20):
    print(i, dictionary[i])<jupyter_output>0 aafc
1 aafc_paca
2 account
3 aggregates
4 agriculture
5 annual
6 average
7 canada
8 canadian
9 comprised
10 constant
11 covering
12 data
13 database
14 defined
15 documentation
16 dollar
17 estimate
18 estimated
19 factor<jupyter_text>Corpus<jupyter_code>VECTOR_SIZE=50
lda_model:LdaModel = LdaModel(corpus, 
                              num_topics=VECTOR_SIZE, 
                              passes=4)
lda_model.num_topics
text = catalog.description_en[0]
tokens = text_to_tokens(text)
bag_of_words = dictionary.doc2bow(tokens)
pd.DataFrame(lda_model[bag_of_words], 
             columns=['Topic','Relevance']).set_index('Topic')
def topic_vector(topic_model:LdaModel, text:str):
    processed_text = text_to_tokens(text)
    bag_of_words = dictionary.doc2bow(processed_text)
    fingerprint = [0] * topic_model.num_topics
    for topic, prob in topic_model[bag_of_words]:
        fingerprint[topic] = prob
    return fingerprint
import matplotlib.pyplot as plt
import matplotlib.style as style
from IPython.display import display

style.use('fivethirtyeight')

VECTOR_SIZE=50
%matplotlib inline

def show_fingerprint(topic_model, text:str):
    display(text)
    vector = topic_vector(topic_model, text)
    plt.figure(figsize=(14,2))
    ax = plt.bar( range(len(vector)), 
                 vector, 
                 0.25, 
                 linewidth=1)
    plt.ylim(top=0.4)
    plt.tick_params(axis='both', 
                    which='both',
                    left=False, 
                    bottom=False, 
                    top=False,
                    labelleft=False, 
                    labelbottom=False)
    plt.grid(False)
show_fingerprint(lda_model, catalog.description_en[0])
show_fingerprint(lda_model, catalog.description_en[3])
show_fingerprint(lda_model, catalog.description_en[2])
topic_vector(lda_model, catalog.description_en[0])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09069373942470389, 'fraction_numerical': 0.020642978003384094, 'mean_word_length': 4.07032590051458, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '459936', 'license': 'MIT', 'n_tokens_mistral': 1010, 'n_tokens_neox': 962, 'n_words': 183, 'path': 'Lesson5/Exercise50.ipynb', 'repo_name': 'TrainingByPackt/Introduction-to-Natural-Language-Processing', 'size': '2955'}