<jupyter_start><jupyter_text>Data Analysis<jupyter_code>from gensim.corpora import Dictionary
from gensim.models import Phrases, TfidfModel
from gensim.models.coherencemodel import CoherenceModel
from gensim.models.ldamodel import LdaModel
from gensim.models.phrases import Phraser
# import math
from matplotlib import pyplot as plt
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize
import pandas as pd
import pickle
import re
# from sklearn.cluster import KMeans
# from sklearn.feature_extraction.text import TfidfVectorizer
# from sklearn.mixture import GaussianMixture
pd.options.mode.chained_assignment = None
REVIEWS = './data/cellphone_reviews.json'<jupyter_output><empty_output><jupyter_text>Utils<jupyter_code>def flatten(lol):
    return [l for ll in lol for l in ll]<jupyter_output><empty_output><jupyter_text>Pre-processing<jupyter_code>reviews = pd.read_json(REVIEWS, lines=True)
reviews['unhelpful'] = reviews['helpful'].apply(lambda x: x[1] - x[0])
reviews['helpful'] = reviews['helpful'].apply(lambda x: x[0])
reviews['reviewText'] = reviews['reviewText'].str.lower()
reviews.drop_duplicates(inplace=True)
reviews.head(5)<jupyter_output><empty_output><jupyter_text>Filtering for negative reviews<jupyter_code>negative_reviews = reviews.loc[reviews['overall'] <= 2]
negative_reviews.head(5)<jupyter_output><empty_output><jupyter_text>Feature extraction<jupyter_code>STOP_WORDS = set(stopwords.words('english'))
STOP_WORDS -= {'not', 'no'}

def tokenizer(sentence):
    tokens = [re.sub('[\W_]+', '', word) \
        for word in word_tokenize(sentence) \
            if len(word) > 2 and word not in STOP_WORDS]
    return tokens

def ngram(sent, n):
    """
    Splits a sentence into n-grams.
    """
    # Split sentence into words
    tokens = tokenizer(sent)
    # Zip n consecutive elements into tuples
    ngram_toks = zip(*[tokens[i:] for i in range(n)])
    # Concat
    ngrams = [' '.join(tok) for tok in ngram_toks]
    filtered_ngrams = [ngram for ngram in ngrams \
        if len(ngram) > 2]
    return filtered_ngrams
corpus = negative_reviews['reviewText'].values
all_sentences = flatten([sent_tokenize(review) for review in corpus])
all_valid_words = [tokenizer(sent) for sent in all_sentences]<jupyter_output><empty_output><jupyter_text>Building n-gram modelsAbandoned because `gensim` is not giving any phrases, mostly unigrams.<jupyter_code># Check bigrams
bigrams = Phrases(all_valid_words, min_count=3)
bigram_mdl = Phraser(bigrams)
for avw in all_valid_words[:50]:
    print(bigram_mdl[avw])<jupyter_output>['worked', 'first', 'week', 'charge', 'phone']
['waste_money']
['worked_great', 'first', 'couple_weeks', 'stopped', 'completely', 'basically', 'small', 'waste_money']
['disappointed', 'nt', 'work', 'ipad']
['get', 'buying', 'cheap', 'adapter']
['week', 'one', 'side', 'works']
['works', 'one', 'side', 'time']
['connect', 'two', 'cables', 'one', 'side', 'stop_working', 'also', 'overheated', 'burning', 'fuses']
['purchased', 'two', 'problem']
['cheap', 'bad', 'quality']
['nt', 'last_long']
['worked_great', 'worked', 'cheap', 'piece_plastic', 'crap', 'nt', 'expected', 'last']
['bought', 'could', 'use', 'charge', 'tab', 'time']
['tab', 'not', 'recognize', 'high', 'power', 'port', 'either', 'charge', 'use', 'power', 'not', 'charge', 'powered']
['could', 'give', 'usb', 'car_charger', 'stars', 'although', 'worked_fine', 'months', 'subsequently', 'died', 'mepros', 'has', 'usb_ports', 'charging', 'one', 'top', '21_amps']
['bottom', 'slot', 'lower', 'presumably', '15', 'ampsfits', 'well', 'chargi[...]<jupyter_text>N-gram generation<jupyter_code>negative_reviews.loc[:, 'reviewSents'] = negative_reviews['reviewText'] \
    .apply(sent_tokenize)
negative_reviews['unigrams'] = negative_reviews['reviewSents'] \
    .apply(lambda sents: flatten([ngram(sent, 1) for sent in sents]))
negative_reviews['bigrams'] = negative_reviews['reviewSents'] \
    .apply(lambda sents: flatten([ngram(sent, 2) for sent in sents]))
negative_reviews['ubgrams'] = negative_reviews['unigrams'] \
    + negative_reviews['bigrams']
negative_reviews['trigrams'] = negative_reviews['reviewSents'] \
    .apply(lambda sents: flatten([ngram(sent, 3) for sent in sents]))
negative_reviews['ngrams'] = negative_reviews['unigrams'] \
    + negative_reviews['bigrams'] \
    + negative_reviews['trigrams']
negative_reviews.head(5)<jupyter_output><empty_output><jupyter_text>Topic Inference  Estimating no. of topics (ngrams)> **Note:** Decreasing `u_mass` coherence values for all models,   using `c_v` metric as measure instead. Also considered k-means  based on TF-IDF, but that's "hard" clustering.<jupyter_code>def get_best_model(texts, max_topics, step=1):
    dictionary = Dictionary(texts)
    lda_corpus = [dictionary.doc2bow(text) for text in texts]
    est_topics = range(2, max_topics + step, step)
    lda_mdls = []
    coh_vals = []
    for i in est_topics:
        lda_mdl = LdaModel(lda_corpus, num_topics=i, id2word=dictionary,
            passes=3, alpha=[0.01] * i, eta=[0.01] * len(dictionary.keys()))
        lda_mdls.append(lda_mdl)
        coh_mdl = CoherenceModel(model=lda_mdl, texts=texts, 
            corpus=lda_corpus, dictionary=dictionary, coherence='c_v')
        coh_vals.append(coh_mdl.get_coherence())
    plt.plot(est_topics, coh_vals)
    plt.xlabel('Number of topics')
    plt.ylabel('Coherence values')
    plt.title('Elbow curve')
    plt.show()
    best_mdl = None
    best_cv = 0
    for idx, cv in enumerate(coh_vals):
        if cv > best_cv:
            best_cv = cv
            best_mdl = lda_mdls[idx]
    return best_mdl<jupyter_output><empty_output><jupyter_text>Ngram version has decreasing `c_v` coherence values, not useful indetermining topic number<jupyter_code>ngram_texts = list(negative_reviews['ngrams'].values)
best_ngram_model = get_best_model(ngram_texts, 20, 2)<jupyter_output><empty_output><jupyter_text>Too little trigrams occur in multiple reviews, cannot infer topicfrom topic words (overlaps).<jupyter_code>trigram_texts = list(negative_reviews['trigrams'].values)
best_trigram_model = get_best_model(trigram_texts, 20, 2)<jupyter_output><empty_output><jupyter_text>A LDA model based on unigrams + bigrams with ~8 topics seems to be the best-performing model.<jupyter_code>ubgram_texts = list(negative_reviews['ubgrams'].values)
best_ubgram_model = get_best_model(ubgram_texts, 20, 2)<jupyter_output><empty_output><jupyter_text>Optimal number of clusters extending to beyond 20, but doesn'tseem realistic.<jupyter_code>bigram_texts = list(negative_reviews['bigrams'].values)
best_bigram_model = get_best_model(bigram_texts, 20, 2)<jupyter_output><empty_output><jupyter_text>Lower coherence values in general.<jupyter_code>unigram_texts = list(negative_reviews['unigrams'].values)
best_unigram_model = get_best_model(unigram_texts, 20, 2)<jupyter_output><empty_output><jupyter_text>Looking at topic words<jupyter_code>NUM_TOPICS = 8
ubgram_texts = list(negative_reviews['ubgrams'].values)
dictionary = Dictionary(ubgram_texts)
lda_corpus = [dictionary.doc2bow(text) for text in ubgram_texts]
best_lda_mdl = LdaModel(lda_corpus, num_topics=NUM_TOPICS, id2word=dictionary,
    passes=5, alpha=[0.01] * NUM_TOPICS, eta=[0.01] * len(dictionary.keys()))
best_lda_mdl.show_topics(num_topics=NUM_TOPICS, num_words=15)
best_lda_mdl.save('./reviews/model/lda_mdl.mm')<jupyter_output><empty_output><jupyter_text>Determining reasonsManual topic labelling, still problems with topic quality<jupyter_code>TOPIC_LABELS = {
    0: 'cheap product, waste of money',
    1: 'no protection',
    2: 'faulty charging cable',
    3: 'screen protector bubbles',
    4: 'lousy sound quality',
    5: 'does not work',
    6: 'poor battery life',
    7: 'case does not fit'
}<jupyter_output><empty_output><jupyter_text>Applying LDA Model<jupyter_code>def review_topic(lda_mdl, dictionary, review_text):
    topic = None
    prob = 0
    for t, p in lda_mdl[dictionary.doc2bow(review_text)]:
        if p > prob:
            topic = t
    return topic
negative_reviews['topic_no'] = negative_reviews['ubgrams'] \
    .apply(lambda x: review_topic(best_lda_mdl, dictionary, x))
topic_freq = pd.DataFrame(negative_reviews \
    .groupby(['asin', 'topic_no'])['reviewText'] \
    .agg(['count', list]))
display(topic_freq.head(5))
topic_pcts = pd.merge(topic_freq, topic_freq.groupby(level=0)['count'] \
    .apply(lambda x: x / x.sum()).rename('perc'),
    left_index=True, right_index=True) \
    .reset_index()
topic_pcts['topic'] = topic_pcts['topic_no'] \
    .apply(lambda x: TOPIC_LABELS[x])
topic_pcts.head(5)<jupyter_output><empty_output><jupyter_text>Pickling output<jupyter_code>review_dict = {}
for _, row in topic_pcts.iterrows():
    pdt_asin = str(row['asin'])
    pdt_dict = {'reason': row['topic'], 
        'frequency': row['perc'], 
        'reviews': row['list']}
    if pdt_asin not in review_dict:
        review_dict[pdt_asin] = [pdt_dict]
    else:
        review_dict[pdt_asin].append(pdt_dict)
with open('./reviews/saved_topics.pkl', 'wb') as out_pkl:
    pickle.dump(review_dict, out_pkl)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14338879159369528, 'fraction_numerical': 0.008099824868651488, 'mean_word_length': 5.869924812030075, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '428557', 'license': 'MIT', 'n_tokens_mistral': 3442, 'n_tokens_neox': 3276, 'n_words': 666, 'path': 'analysis.ipynb', 'repo_name': 'jzheng13/saleswhale-ai', 'size': '9136'}