<jupyter_start><jupyter_text>Create our own categories<jupyter_code>category=('wired','gaming','mechanical','ergonomic','membrane','backlit')

for cat in category:
    df[cat]=df['Name'].str.lower().apply(lambda x:bool(re.search(cat,x)) )
df.head()
df=df.iloc[:,5:]
df.head()
df['Time']=df['Time'].str.split('on ',expand=True)[1]

df['Time']=pd.to_datetime(df['Time'])
df.head()
df['Comment']=df['Title']+df['Review'].str.strip()
del df['Title'],df['Review']
df.head()
df.info()
df.dropna(inplace=True)
df.info()
# !pip install langdetect
import langdetect
from langdetect import detect
df['lan']=df['Comment'].apply(lambda x:detect(x))
df['lan'].value_counts()
df=df[df['lan']=='en']
df['lan'].value_counts()
del df['lan']
df.head()
import nltk
from nltk import WordNetLemmatizer
import numpy as np
cps=np.array(df['Comment'])
stop_words=nltk.corpus.stopwords.words('english')
wpt=nltk.WordPunctTokenizer()
 for x in stop_words:
        if x==['no','nor','not']:
            stop_words.remove(x)
        if bool(re.search('n\'t',x)):
            stop_words.remove(x)
stop_words
def normalize_docuement (doc):
    #special char  and number and handling ' for negative words
    doc=re.sub('\'', '',doc) 
    doc=re.sub('[^a-zA-Z]', ' ',doc)
    
    #single char
    doc=re.sub(r'\s+[a-zA-Z]\s+',' ',doc) 
    doc=re.sub(r'^[a-zA-Z]\s+', ' ', doc, re.I|re.A) 
    
    doc=re.sub(r'\s+',' ',doc).lower() # extra white space and lowercase
    #tokenize document
    tokens=wpt.tokenize(doc)
    #filter stopwords out of document
    filtered_tokens=[token for token in tokens if token not in stop_words]
   # doc=' '.join(filtered_tokens)
    
    #lemmentize
    lemmatizer = WordNetLemmatizer()
    newtokens = [lemmatizer.lemmatize(filtered_token) for filtered_token in filtered_tokens]
    #re-create document
    doc=' '.join(newtokens)
    
    return doc
normalize_corpus=np.vectorize(normalize_docuement) 

normalized_cps=normalize_corpus(cps)
from sklearn.decomposition import LatentDirichletAllocation

LDA = LatentDirichletAllocation(n_components=5,random_state=42)
# tfidf

from sklearn.feature_extraction.text import TfidfVectorizer
tv=TfidfVectorizer (min_df=10, max_df=0.95, norm='l2', 
                  use_idf=True, smooth_idf=True)
tv_matrix=tv.fit_transform(normalized_cps)
tv_matrix=tv_matrix.toarray()
vocab=tv.get_feature_names()
pd.DataFrame (np.round(tv_matrix,2), columns=vocab)
LDA.fit(tv_matrix)
for index,topic in enumerate(LDA.components_):
    print(f'THE TOP 10 WORDS FOR TOPIC #{index}')
    print([tv.get_feature_names()[i] for i in topic.argsort()[-10:]])
    print('\n')
#BOW
from sklearn.feature_extraction.text import CountVectorizer

cv = CountVectorizer(min_df=10, max_df=0.95)

cv_matrix=cv.fit_transform(normalized_cps)

cv_matrix=cv_matrix.toarray()

#get all unique words in the corpus
vocab=cv.get_feature_names()
#show document feature matrix
pd.DataFrame(cv_matrix,columns=vocab)
LDA.fit(cv_matrix)
for index,topic in enumerate(LDA.components_):
    print(f'THE TOP 10 WORDS FOR TOPIC #{index}')
    print([cv.get_feature_names()[i] for i in topic.argsort()[-10:]])
    print('\n')
LDA.fit(tv_matrix)
for index,topic in enumerate(LDA.components_):
    print(f'THE TOP 10 WORDS FOR TOPIC #{index}')
    print([tv.get_feature_names()[i] for i in topic.argsort()[-10:]])
    print('\n')
bv = CountVectorizer(ngram_range=(3,3),min_df=2,max_df=0.95)
bv_matrix=bv.fit_transform(normalized_cps)
bv_matrix=bv_matrix.toarray()
vocab=bv.get_feature_names()
pd.DataFrame(bv_matrix,columns=vocab)
#bv -3grams
LDA.fit(bv_matrix)
for index,topic in enumerate(LDA.components_):
    print(f'THE TOP 10 WORDS FOR TOPIC #{index}')
    print([bv.get_feature_names()[i] for i in topic.argsort()[-10:]])
    print('\n')
bv = CountVectorizer(ngram_range=(4,4),min_df=2,max_df=0.95)
bv_matrix=bv.fit_transform(normalized_cps)
bv_matrix=bv_matrix.toarray()
vocab=bv.get_feature_names()
pd.DataFrame(bv_matrix,columns=vocab)
#bv -4grams
LDA.fit(bv_matrix)
for index,topic in enumerate(LDA.components_):
    print(f'THE TOP 10 WORDS FOR TOPIC #{index}')
    print([bv.get_feature_names()[i] for i in topic.argsort()[-10:]])
    print('\n')
topic_results= LDA.transform(bv_matrix)
df['Topic'] = topic_results.argmax(axis=1)
df.head()
from nltk.sentiment.vader import SentimentIntensityAnalyzer

sid = SentimentIntensityAnalyzer()
df['scores'] = df['Comment'].apply(lambda review: sid.polarity_scores(review))
df['compound']  = df['scores'].apply(lambda score_dict: score_dict['compound'])
df['comp_category'] = df['compound'].apply(lambda c: 'pos' if c >=0 else 'neg')

df.head()
df['comp_category'].value_counts()
df.groupby('Topic').mean()['compound']
df.groupby('Topic').count()['compound'].plot.bar(title='Topic distribution').set_ylabel('count');
df[df['comp_category']=='neg']['Topic'].value_counts().plot.bar(title='Negative topic distribution').set_ylabel('count')
df.iloc[:,7:]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1702127659574468, 'fraction_numerical': 0.01094224924012158, 'mean_word_length': 5.961918194640338, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '419444', 'license': 'Apache-2.0', 'n_tokens_mistral': 1930, 'n_tokens_neox': 1836, 'n_words': 321, 'path': 'amazon-sentiment-analysis-lda.ipynb', 'repo_name': 'Calvin-Truong/Keyboard-Sentiment-Analysis', 'size': '4935'}