<jupyter_start><jupyter_text>IR  Dataset handler<jupyter_code>import pandas as pd
# LINK: https://www.kaggle.com/hacker-news/hacker-news-posts
dataset = pd.read_csv('HN_posts_year_to_Sep_26_2016.csv', nrows=10000, usecols=['title'])
dataset.head()
dataset.info()
import nltk
nltk.download('punkt')
from nltk.tokenize import word_tokenize
import matplotlib.pyplot as plt
texts = list(dataset['title'].values)
lens = list(map(lambda x: len(word_tokenize(x)), texts))
plt.hist(lens)<jupyter_output><empty_output><jupyter_text>Preprocessing  Removing stopwords<jupyter_code>import json
import requests
url = "https://countwordsfree.com/stopwords/english/json"
response = pd.DataFrame(data = json.loads(requests.get(url).text), columns=['words'])
SW = list(response['words'])
texts = list(map(lambda x: ' '.join([word for word in word_tokenize(x) if word not in (SW)]), texts))<jupyter_output><empty_output><jupyter_text>Stemming<jupyter_code>from nltk.stem.snowball import SnowballStemmer
stemmer = SnowballStemmer('english')
texts = list(map(lambda x:' '.join([stemmer.stem(y) for y in word_tokenize(x)]), texts))<jupyter_output><empty_output><jupyter_text>Vectorize<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(texts, min_df=2, ngram_range=(1,3), max_features=20000)
tfidf.fit(texts)
vectorize_texts = tfidf.transform(texts)
vectorize_texts.shape<jupyter_output><empty_output><jupyter_text>Cosine similarities to find suitable news<jupyter_code>from sklearn.metrics.pairwise import linear_kernel
articles = list(dataset['title'].values)
query = 'webrtc'
query = list(map(lambda x:' '.join([stemmer.stem(y) for y in word_tokenize(x)]), [query]))
new_features = tfidf.transform(query)
cosine_similarities = linear_kernel(new_features, vectorize_texts).flatten()
related_docs_indices = cosine_similarities.argsort()[::-1]

topk = 5
print(f"Top-{topk} documents")
for i in range(topk):
  print(i + 1, articles[related_docs_indices[i]])<jupyter_output>Top-5 documents
1 WebRTC Weekly
2 WebRTC that will help you cook better
3 Welcome Kurento WebRTC Team to Twilio
4 Wowza Media Streaming Engine Announces Support for WebRTC (Beta)
5 Ask HN: WebRTC onion routing between Facebook friend circles<jupyter_text>Topic modeling  Dataset handler<jupyter_code>import pandas as pd
# LINK: https://www.kaggle.com/c/learn-ai-bbc/data
dataset = pd.read_csv('BBC News Train.csv')
dataset.info()
dataset = dataset.drop('ArticleId', axis=1)
dataset
corpus = dataset['Text'].values<jupyter_output><empty_output><jupyter_text>Preprocessing  Tokenize<jupyter_code>import nltk
import re

regex = re.compile('[^a-zA-Z]')

def tokenize(text):
    tokens = []
    for sent in nltk.sent_tokenize(text):
        for word in nltk.word_tokenize(sent):
            clean_word = regex.sub('', word)
            tokens.append(clean_word.lower())
    return tokens<jupyter_output><empty_output><jupyter_text>Remove stop words<jupyter_code>import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords as sw
stopwords = sw.words('english')<jupyter_output><empty_output><jupyter_text>Stemming<jupyter_code>from nltk.stem.snowball import SnowballStemmer

stemmer = SnowballStemmer("english")

def stem(word):
    return stemmer.stem(word).strip()<jupyter_output><empty_output><jupyter_text>Vectorize<jupyter_code>nltk.download('punkt')
from sklearn.feature_extraction.text import TfidfVectorizer

tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=20000,
                                 min_df=0.05, stop_words=stopwords,
                                 use_idf=True, tokenizer=tokenize,
                                  lowercase=True, preprocessor=stem)
tfidf_matrix = tfidf_vectorizer.fit_transform(corpus)<jupyter_output><empty_output><jupyter_text>LDA<jupyter_code>from sklearn.decomposition import LatentDirichletAllocation

lda = LatentDirichletAllocation(n_components=5)
lda.fit(tfidf_matrix)
for i,topic in enumerate(lda.components_):
    print(f'Topic #{i}:')
    print([tfidf_vectorizer.get_feature_names()[i] for i in topic.argsort()[-10:]])
    print('\n')
topic_values = lda.transform(tfidf_matrix)
doc_num, topic_num = topic_values.shape
from sklearn.decomposition import LatentDirichletAllocation

lda = LatentDirichletAllocation(n_components=5)
lda.fit(tfidf_matrix)import pandas as pd
df = pd.DataFrame({'document': corpus, 'label': dataset['Category'], 'lda': topic_values.argmax(axis=1)})
df.groupby(['label', 'lda']).count()
from sklearn.decomposition import LatentDirichletAllocation

lda = LatentDirichletAllocation(n_components=5, random_state=2021)
lda.fit(tfidf_matrix)
for i,topic in enumerate(lda.components_):
    print(f'Topic #{i}:')
    print([tfidf_vectorizer.get_feature_names()[i] for i in topic.argsort()[-10:]])
    print('\n')<jupyter_output>Topic #0:
['tv', 'films', 'music', 'star', 'actor', 'show', 'award', 'awards', 'best', 'film']


Topic #1:
['use', 'net', 'digital', 'phone', 'music', 'technology', 'users', 'software', 'mobile', 'people']


Topic #2:
['government', 'party', 'win', 'election', 'blair', 'england', 'game', 'labour', 'would', 'mr']


Topic #3:
['shares', 'economy', 'growth', 'bank', 'year', 'market', 'company', 'firm', 'us', 'bn']


Topic #4:
['december', 'problem', 'american', 'match', 'november', 'comes', 'leading', 'event', 'open', 'injury']
{'fraction_non_alphanumeric': 0.14275023386342375, 'fraction_numerical': 0.01047708138447147, 'mean_word_length': 6.59375, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '419693', 'license': 'MIT', 'n_tokens_mistral': 1986, 'n_tokens_neox': 1900, 'n_words': 363, 'path': 'code/TSLm.ipynb', 'repo_name': 'TokisakiKurumi2001/GDSC_TSLm', 'size': '5345'}