<jupyter_start><jupyter_text>Non-Negative Matrix Factorization Reuter data<jupyter_code>import time 
import pandas as pd
import numpy as np
import glob
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF
from collections import Counter
import nltk
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
import re
import warnings 
warnings.filterwarnings('ignore')
tic = time.time()
path = r'/Users/alex/Desktop/training'
all_files = glob.glob(path + "/*.txt")

li = []

for file in all_files:
    df = pd.read_csv(file, sep='\n')
    li.append(df)

toc = time.time()
print(toc - tic)
def RetrievingText(datalist = li):
    return [''.join(li[i].iloc[:,0]) for i in range(len(li))]

dic = dict(head = [df.columns[0] for df in li], 
           text = RetrievingText())
df = pd.DataFrame(dic)
df.head()
pd.set_option('display.max_colwidth', -1)
df.head(2)
train, test = train_test_split(df, test_size = 0.2, random_state = 123)
stemmer = nltk.stem.SnowballStemmer('english')
stopwords = set(stopwords.words('english'))
def tokenize(text, lower=True):
    
    regex1 = re.compile('[^a-zA-Z]')
    if lower:
        corpus = regex1.sub(' ', text.lower())
        words = [stemmer.stem(word) for word in word_tokenize(corpus) if word not in stopwords]
    else:
        corpus = regex1.sub(' ', text.lower())
        words = [stemmer.stem(word) for word in word_tokenize(corpus) if word not in stopwords]
    
    return words 
params = {
    'tokenizer' : tokenize, 
    'stop_words' : None, 
    'max_df' : 0.75, 
    'max_features' : 1000, 
    'lowercase' : False
}

vectorizer = TfidfVectorizer(**params)
tfidf_vect = vectorizer.fit_transform(train.text)
tfidf_vect.A
vectorizer.get_feature_names()[:10]
clf = NMF(n_components = 10, random_state=123)
W1 = clf.fit_transform(tfidf_vect)
H1 = clf.components_
print(f'W matrix consists of {W1.shape}')
print(f'H matrix consists of {H1.shape}')
num_words = 15

vocab = np.array(vectorizer.get_feature_names())

top_words = lambda t:[vocab[i] for i in np.argsort(t)[:-num_words-1:-1]] # 15 words from the right side 
topic_words = ([top_words(t) for t in H1]) 
topics = [' '.join(t) for t in topic_words]
vocab = np.array(vectorizer.get_feature_names())
top_words(H1[0])
topics
import pyLDAvis
from pyLDAvis import sklearn as skvis
from sklearn.feature_extraction.text import CountVectorizer
df.text.head(1)
# without setting parameters
bow0 = CountVectorizer()
bow0.fit(df.text)
import itertools 
dict(itertools.islice(bow0.vocabulary_.items(), 10))
transformed_text0 = bow0.transform(df.text)
def tokenize(text, lower=True):
    
    regex1 = re.compile('[^a-zA-Z]')
    if lower:
        corpus = regex1.sub(' ', text.lower())
        words = [stemmer.stem(word) for word in word_tokenize(corpus) if word not in stopwords]
    else:
        corpus = regex1.sub(' ', text.lower())
        words = [stemmer.stem(word) for word in word_tokenize(corpus) if word not in stopwords]
    
    return words 
config = {
    'tokenizer' : tokenize, 
    'stop_words' : None, 
    'max_df' : 0.75, 
    'max_features' : 1000, 
    'lowercase' : False
}
bow = CountVectorizer()
bow.set_params(**config)
bow.fit(df.text)

dict(itertools.islice(bow.vocabulary_.items(),10))
transformed_text = bow.transform(df.text)
transformed_text.A
#NMF
nmf0 = NMF(n_components = 15, random_state=1234)
nmf0.fit(transformed_text0)
model = skvis.prepare(nmf0, transformed_text0, bow0)
pyLDAvis.display(model)
def topic_words(clf, vectorizer, n_words = 15):
    vocabulary = np.array(vectorizer.get_feature_names())
    X = clf.components_
    top_words = lambda t: [vocabulary[i] for i in np.argsort(t)[:-n_words-1:-1]]
    topic_words = ([top_words(t) for t in X])
    return [' '.join(t) for t in topic_words]

topic_words(nmf0, bow0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13182049974502805, 'fraction_numerical': 0.01963284038755737, 'mean_word_length': 4.322930800542741, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '17616', 'license': 'MIT', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1435, 'n_words': 333, 'path': 'NMF/NMF_Reuters21578.ipynb', 'repo_name': 'hshehjue/Natural_Language_Processing', 'size': '3922'}