<jupyter_start><jupyter_text>Import libraries<jupyter_code>import pandas as pd
import spacy
import nltk
import re
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.model_selection import train_test_split
import numpy as np
import pickle
from __future__ import print_function
import pyLDAvis
import pyLDAvis.sklearn
pyLDAvis.enable_notebook()
%matplotlib inline
import seaborn as sns
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
nlp = spacy.load("en_core_web_sm")
random_state=695
# a little helper function to load model from pkl file
def load_model(x):
    infile = open(x,'rb')
    lda_model = pickle.load(infile)
    infile.close()
    return lda_model<jupyter_output><empty_output><jupyter_text>Import reviews<jupyter_code>with open('sg_reviews.pkl', 'rb') as f:
    sg_reviews = pickle.load(f)
    
with open('au_reviews.pkl', 'rb') as s:
    au_reviews = pickle.load(s)
sg_reviews['review']=sg_reviews['review'].astype(str)
au_reviews['review']=au_reviews['review'].astype(str)<jupyter_output><empty_output><jupyter_text>Pre-processing<jupyter_code>stop_words=spacy.lang.en.stop_words.STOP_WORDS
common_phrase=['food panda', 'panda', 'deliveroo', 'doordash', 'ubereats', 'ubereat', 'menulog', 'grubhub', 'grabfood', 'good', 'nice',
              'excellent', 'ok','amazing', 'great']
stop_words.update(common_phrase)

def spacy_tokenizer(tokens):
    
     
    # lemmetize and remove punctuation
    lemma_tokens=[token.lemma_.lower().strip() for token in tokens if not token.is_punct]
    
    # removing stop words
    lemma_tokens_no_stop_words=[word for word in lemma_tokens if word not in stop_words]
    
      
    return lemma_tokens_no_stop_words

def preprocess_pipe(text_col, noun=False):
    preproc_pipe = []
    if noun==False:
        for doc in nlp.pipe(text_col, disable=['parser', 'ner'], n_process=15, batch_size=2000):
            preproc_pipe.append(spacy_tokenizer(doc))
    else:
        for doc in nlp.pipe(text_col, disable=['ner'], n_process=15, batch_size=2000):
            preproc_pipe.append([n.text for n in doc.noun_chunks])
    return preproc_pipe

%%time
print("start preprocessing sg")
sg_reviews['review_tok']=preprocess_pipe(sg_reviews['review'])
print("start preprocessing au")
au_reviews['review_tok']=preprocess_pipe(au_reviews['review'])

print('finish preprocessing')

sg_pos_reviews=sg_reviews[sg_reviews['rating']>2]
sg_neg_reviews=sg_reviews[sg_reviews['rating']<3]
au_pos_reviews=au_reviews[au_reviews['rating']>2]
au_neg_reviews=au_reviews[au_reviews['rating']<3]

all_pos_reviews=pd.concat([sg_pos_reviews, au_pos_reviews], ignore_index=True)
all_neg_reviews=pd.concat([sg_neg_reviews, au_neg_reviews], ignore_index=True)<jupyter_output>start preprocessing sg
start preprocessing au
finish preprocessing
CPU times: total: 7min 26s
Wall time: 8min 2s<jupyter_text>Sickit-learn - Model Training<jupyter_code>#tfidf vectorization
def dummy(x):
    return x

def vectorization(text, spacy=True, split=True):
    
    if split==True:
        X_train, X_test=train_test_split(text, test_size=0.1, random_state=random_state)
    else:
        X_train=text
        vectorized_data_test=None
    
    if spacy==True:
        tfidf_vectorizer = TfidfVectorizer(ngram_range = (1,1),
                                           min_df=100,
                                           analyzer='word',
                                           tokenizer=dummy,
                                           preprocessor=dummy,
                                           token_pattern=None)        
    else:
        tfidf_vectorizer = TfidfVectorizer(ngram_range = (1,1), lowercase=True, min_df=100, stop_words='english')

    tfidf_documents = tfidf_vectorizer.fit_transform(X_train)
    
    tfidf_feature_names = tfidf_vectorizer.get_feature_names()
    
    if split==True:
        vectorized_data_test=tfidf_vectorizer.transform(X_test)
    
    return tfidf_documents, tfidf_feature_names, tfidf_vectorizer, vectorized_data_test
def lda_training(n_topic, tfidf_doc, feature_names, test_data=None):

    lda = LatentDirichletAllocation(n_components = n_topic,
                                    learning_method='online',
                                    random_state=random_state,
                                    batch_size=128,
                                    evaluate_every = -1,
                                    n_jobs = 15)
    lda.fit(tfidf_doc)
    topic_models = lda.components_
    if test_data != None:
        # Evaluation Higher the better
        print("Log Likelihood: ", lda.score(test_data))

        # Evaluation Lower the better
        print("Perplexity: ", lda.perplexity(test_data))
    
    #Display top 10 words for each topic
    no_top_words = 10
    
    for topic_idx, topic in enumerate(lda.components_):
        term_list = [feature_names[i]
                        for i in topic.argsort()[:-no_top_words - 1:-1]]
        print("topic %d:" % (topic_idx), term_list)
    
    return lda<jupyter_output><empty_output><jupyter_text>Start with 10% of the data to search the best value for n_components.<jupyter_code>%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
sg_sklearn_model=lda_training(2, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(sg_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
sg_sklearn_model=lda_training(3, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(sg_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
sg_sklearn_model=lda_training(4, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(sg_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
sg_sklearn_model=lda_training(5, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(sg_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
sg_sklearn_model=lda_training(6, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(sg_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
sg_sklearn_model=lda_training(7, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(sg_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
sg_sklearn_model=lda_training(8, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(sg_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
sg_sklearn_model=lda_training(9, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(sg_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
sg_sklearn_model=lda_training(10, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(sg_sklearn_model, tfidf_documents, vectorizer)<jupyter_output>Log Likelihood:  -56936.83348742516
Perplexity:  866.899473811719
topic 0: ['app', 'order', 'far', 'hour', 'add', 'wait', 'cancel', 'offer', 'deliver', 'lot']
topic 1: ['easy', 'use', 'fast', 'convenient', 'helpful', 'foodpanda', 'rider', 'slow', 'n', 'poor']
topic 2: ['awesome', 'like', 'app', 'grab', 'order', 'thank', 'friendly', 'user', 'food', 'payment']
topic 3: ['help', 'customer', '', 'address', '😍', 'nt', 'expensive', 'try', 'company', 'hungry']
topic 4: ['experience', 'work', 'time', 'update', 'voucher', 'app', 'need', 'restaurant', 'superb', 'fee']
topic 5: ['love', '👍', 'bad', 'job', 'app', '️', '😊', 'u', 'terbaik', 'di']
topic 6: ['late', 'reliable', 'error', 'wow', 'happy', 'quality', 'gud', 'bug', 'fine', 'stupid']
topic 7: ['driver', 'charge', '❤', '👌', 'promo', 'number', 'option', 'support', 'code', 'phone']
topic 8: ['service', 'improve', 's', 'feature', 'bagus', 'improvement', 'mobile', 'need', '4', 'edit']
topic 9: ['food', 'delivery', 'order', 'app', 'useful', 'disc[...]<jupyter_text>The best parameter for n_component looks like is 8 for singgapore dataset! lets train the LDA model using the full dataset. Since this training process will take more than 30 mins, we are loading the trained model in below, but the code is there is you want to reproduce the result<jupyter_code>%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(sg_reviews.review_tok, spacy=True, split=False)
#best_sg_sklearn_model=lda_training(8, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(load_model('best_sg_sklearn_model.pkl'), tfidf_documents, vectorizer)
# print out the top 5 reviews that belong to each topic, to have a better understanding on the context
sg_tfidf_documents, sg_feature_names=vectorization(sg_reviews.review_tok, spacy=True, split=False)[:2]

sg_doc_topic=best_sg_sklearn_model.transform(sg_tfidf_documents)
sg_topic_weights=pd.DataFrame(sg_doc_topic, columns=["topic_{}".format(n) for n in range(8)])
sg_topic_weights['review']=sg_reviews.reset_index().review

for n in range(8):
    topic = best_sg_sklearn_model.components_[n]
    term_list = [sg_feature_names[i] for i in topic.argsort()[:-10 - 1:-1]]
    print("topic_{}:".format(n), term_list)
    for review in sg_topic_weights.sort_values('topic_{}'.format(n), ascending=False).head().review.to_list():
        print(review)
        print('-----------------------------')
    print('')<jupyter_output>topic_0: ['order', 'food', 'time', 'helpful', 'service', 'customer', 'delivery', 'deliver', 'app', 'driver']
I used grab app many years, this experience really make me fed up..ordered dinner to celebrate Xmas at home, waiting for long time and the food went missing...contacted help center and they say will investigate this issue. However, waiting for two weeks and yet to get any updates. I send email to ask for update few times and not even get any reply. Until I give up I email say if your side couldn’t give any update then please refund me..
Funniest thing is..I got my refund and the status become problem solved. So what means the investigation? I happily ordered food for celebration ended up I received nothing, and this is what grab provided the service?
-----------------------------
Forget about using grab for food delivery when you need it the most - during busy periods or when it rains. Orders placed are stuck at the stage where “it is being accepted” with zero visibility on whet[...]<jupyter_text>Assign themes to each topictopic 0: Order placingtopic 1: app UI/functionstopic 2: Delivery in generaltopic 3: Service reliabilitytopic 4: Unclassifiedtopic 5: Customer servicetopic 6: app UI/functionstopic 7: Delivery speed<jupyter_code># calculate the dominant topic for each review
def topic_labeling(row, topic_columns):
    w_dict={}
    for n in topic_columns:
        w_dict[n]=row[n]
    max_key = max(w_dict, key=w_dict.get)
    return max_key
        
    
sg_topic_weights['dominant_topic_label']=sg_topic_weights.apply(lambda row: topic_labeling(row, sg_topic_weights.columns[:-1]), axis=1)
sg_topic_weights['dominant_topic_label']=sg_topic_weights['dominant_topic_label'].replace({'topic_0': 'Order placing',
                                                                                              'topic_1': 'app UI/functions',
                                                                                              'topic_2': 'General feeling on the services',
                                                                                              'topic_3': 'Service reliability',
                                                                                              'topic_4': 'Unclassified',
                                                                                              'topic_5': 'Customer service',
                                                                                              'topic_6': 'app UI/functions',
                                                                                              'topic_7': 'Delivery speed'})
sg_topic_weights['dominant_topic_label'].value_counts(normalize=True, ascending=True).plot.barh(title='Singapore reviews themes', xlim=(0,0.5))<jupyter_output>c:\users\stanley\appdata\local\programs\python\python39\lib\site-packages\pandas\plotting\_matplotlib\core.py:1437: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
  self.tick_pos = ax.convert_yunits(self.ax_index).astype(np.int)<jupyter_text>Each review may have more than 1 topic, in here we used the topic contribution score average as a threshold. If a review that has a topic score higher than the topic socre mean, we will assign that topic to the review as well<jupyter_code># 
def multiple_labeling(row, col, mean_value):
    if row['dominant_topic_label']==col:
        return True
    elif row[col]>mean_value:
        return True
    else:
        return False
    

for col in tqdm(sg_topic_weights.columns[:8]):
    mean_value=sg_topic_weights[col].mean()
    sg_topic_weights['{}_label'.format(col)]=sg_topic_weights.apply(lambda row: multiple_labeling(row, col, mean_value), axis=1)

y_values=[]
total=sg_topic_weights.shape[0]
for col in sg_topic_weights.columns[-8:]:
    y_values.append(sg_topic_weights[col].value_counts().loc[True]/total)
plt.bar(sg_topic_weights.columns[-8:], y_values)
plt.xticks(fontsize=12, rotation=90)
plt.yticks(fontsize=12 )
plt.show()<jupyter_output><empty_output><jupyter_text>AU reviews topic num search<jupyter_code>%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(au_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
au_sklearn_model=lda_training(2, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(au_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(au_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
au_sklearn_model=lda_training(3, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(au_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(au_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
au_sklearn_model=lda_training(4, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(au_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(au_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
au_sklearn_model=lda_training(5, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(au_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(au_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
au_sklearn_model=lda_training(6, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(au_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(au_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
au_sklearn_model=lda_training(7, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(au_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(au_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
au_sklearn_model=lda_training(8, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(au_sklearn_model, tfidf_documents, vectorizer)
%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(au_reviews.sample(frac=0.1, random_state=random_state).review_tok, spacy=True)
au_sklearn_model=lda_training(9, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(au_sklearn_model, tfidf_documents, vectorizer)<jupyter_output>Log Likelihood:  -80028.13004218457
Perplexity:  1042.8236571669438
topic 0: ['easy', 'use', 'far', 'app', 'reliable', 'let', 'problem', 'need', 'high', 'simple']
topic 1: ['bad', 'service', 'customer', 'door', 'available', 'area', 'app', 'dash', 'deal', 'useless']
topic 2: ['service', 'food', 'time', 'delivery', 'deliver', 'driver', 'pretty', 'guy', 'order', 'quality']
topic 3: ['order', 'app', 'food', 'time', 'uber', 'delivery', 'use', 'driver', 'service', 'customer']
topic 4: ['👍', 'super', 'u', 'zomato', 'de', 'correct', 'issue', 'y', '️', 'swiggy']
topic 5: ['experience', 'late', 'happy', 'perfect', 'fantastic', 'choice', 'wonderful', 'exceptional', 'respond', 'rate']
topic 6: ['fast', 'convenient', 'work', 'friendly', 'offer', 'expensive', 'like', 'user', 'delivery', 'easy']
topic 7: ['app', 'useful', 'cool', 'm', 'slow', 'fee', 'delivery', 'open', 'improve', 'accurate']
topic 8: ['love', 'awesome', 'quick', 'uber', 'eat', '👌', 'hot', 'life', 'easy', 'job']
CPU times: total: 1min[...]<jupyter_text>The best parameter for n_component looks like is 8 for Australia dataset! lets train the LDA model using the full dataset. Since this training process will take more than 30 mins, we are loading the trained model in below, but the code is there is you want to reproduce the result.<jupyter_code>%%time
tfidf_documents, feature_names, vectorizer, vectorized_data_test = vectorization(au_reviews.review_tok, spacy=True, split=False)
#best_au_sklearn_model=lda_training(8, tfidf_documents, feature_names, vectorized_data_test)
pyLDAvis.sklearn.prepare(load_model('best_au_sklearn_model.pkl'), tfidf_documents, vectorizer)
for topic_idx, topic in enumerate(load_model('best_au_sklearn_model.pkl').components_):
    term_list = [feature_names[i]
                    for i in topic.argsort()[:-10 - 1:-1]]
    print("topic %d:" % (topic_idx), term_list)<jupyter_output>topic 0: ['order', 'food', 'time', 'deliver', 'delivery', 'cancel', 'driver', 'service', 'app', 'customer']
topic 1: ['love', 'app', 'brilliant', 'helpful', 'pretty', 'wonderful', 'swiggy', 'gud', '😍', 'accurate']
topic 2: ['awesome', 'quick', 'convenient', '👍', 'thank', 'uber', 'easy', 'eat', 'fantastic', 'efficient']
topic 3: ['app', 'easy', 'use', 'work', 'super', 'number', 'login', 'payment', 'useful', 'simple']
topic 4: ['far', 'friendly', 'like', 'user', 'perfect', 'happy', 'app', 'zomato', 'area', 'available']
topic 5: ['service', 'fast', 'food', 'delivery', 'app', 'reliable', 'superb', 'cool', 'quality', 'poor']
topic 6: ['order', 'delivery', 'app', 'offer', 'restaurant', 'food', 'option', 'nt', 'discount', 'need']
topic 7: ['bad', 'experience', 'customer', 'service', 'app', 'nt', 'support', 'issue', 'terrible', 'n']<jupyter_text>One of the key challenges in topic modeling is to discover the theme from the generated topic. Although the words frequency gave us a proxy of its meaning, it would be better to look at a few documents which are straonly associated with each topic. So that we can have a better understaning.<jupyter_code>au_sklearn_model=load_model('best_au_sklearn_model.pkl')
au_tfidf_documents, au_feature_names=vectorization(au_reviews.review_tok, spacy=True, split=False)[:2]

au_doc_topic=au_sklearn_model.transform(au_tfidf_documents)
au_topic_weights=pd.DataFrame(au_doc_topic, columns=["topic_{}".format(n) for n in range(8)])
au_topic_weights['review']=au_reviews.reset_index().review

for n in range(8):
    topic = au_sklearn_model.components_[n]
    term_list = [au_feature_names[i] for i in topic.argsort()[:-10 - 1:-1]]
    print("topic_{}:".format(n), term_list)
    for review in au_topic_weights.sort_values('topic_{}'.format(n), ascending=False).head().review.to_list():
        print(review)
        print('-----------------------------')
    print('')<jupyter_output>topic_0: ['order', 'food', 'time', 'deliver', 'delivery', 'cancel', 'driver', 'service', 'app', 'customer']
This is the second time this week I’ve received stale and cold food. The first time I placed my order it was to a restaurant 5 minutes away from my house so not a far drive, right. So I placed my order it said it would arrive within 30 minutes at the latest. 50min goes by and I call the dasher because I noticed on the order tracking he was driving all over the place and 20 minutes in the opposite direction to the restaurant. Then after and hour and 20min later I finally got a resolution from their customer service team and they said they’re sending a new driver to pick it up.  I kid you not almost 2 hours later I get my order and it’s cold and stale completely inedible. They did credit me money but why did they not get them to remake the meal if they know how long ago it was ready to be picked up.  Just so you can better understand my frustration by this point I have a meal I was[...]<jupyter_text>topic 0: Order placingtopic 1: Overall app experiencetopic 2: General feeling on the servicetopic 3: app UI/functionstopic 4: Delivery service qualitytopic 5: Delivery speedtopic 6: Deliverytopic 7: Customer service<jupyter_code>def topic_labeling(row, topic_columns):
    w_dict={}
    for n in topic_columns:
        w_dict[n]=row[n]
    max_key = max(w_dict, key=w_dict.get)
    return max_key
        
    
au_topic_weights['dominant_topic_label']=au_topic_weights.apply(lambda row: topic_labeling(row, au_topic_weights.columns[:-1]), axis=1)
au_topic_weights['dominant_topic_label']=au_topic_weights['dominant_topic_label'].replace({'topic_0': 'Order placing',
                                                                                              'topic_1': 'Overall app experience',
                                                                                              'topic_2': 'General feeling on the service',
                                                                                              'topic_3': 'app UI/functions',
                                                                                              'topic_4': 'Delivery service quality',
                                                                                              'topic_5': 'Delivery speed',
                                                                                              'topic_6': 'Delivery',
                                                                                              'topic_7': 'Customer service'})

au_topic_weights['dominant_topic_label'].value_counts(normalize=True, ascending=True).plot.barh(title='Australia reviews themes', xlim=(0,0.5))
def multiple_labeling(row, col, mean_value):
    if row['dominant_topic_label']==col:
        return True
    elif row[col]>mean_value:
        return True
    else:
        return False
    

for col in tqdm(au_topic_weights.columns[:8]):
    mean_value=au_topic_weights[col].mean()
    au_topic_weights['{}_label'.format(col)]=au_topic_weights.apply(lambda row: multiple_labeling(row, col, mean_value), axis=1)

y_values=[]
total=au_topic_weights.shape[0]
for col in au_topic_weights.columns[-8:]:
    y_values.append(au_topic_weights[col].value_counts().loc[True]/total)
plt.bar(au_topic_weights.columns[-8:], y_values)
plt.xticks(fontsize=12, rotation=90)
plt.yticks(fontsize=12 )
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1204351825926963, 'fraction_numerical': 0.01263949442022319, 'mean_word_length': 4.2098353823713275, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 46, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 71, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '437493', 'license': 'MIT', 'n_tokens_mistral': 8187, 'n_tokens_neox': 7860, 'n_words': 1907, 'path': 'playground/stanley/Unsupervised_learning.ipynb', 'repo_name': 'mads-app-reviews-nlp/app-reviews-nlp', 'size': '25001'}