<jupyter_start><jupyter_text>Installing Libraries<jupyter_code>!pip install google_play_scraper
!pip install sklearn
!pip install transformers requests beautifulsoup4 pandas numpy
!pip install torch
!pip install neattext<jupyter_output>Collecting neattext
  Downloading neattext-0.1.2-py3-none-any.whl (114 kB)
[?25l
[K     |██▉                             | 10 kB 22.3 MB/s eta 0:00:01
[K     |█████▊                          | 20 kB 10.2 MB/s eta 0:00:01
[K     |████████▋                       | 30 kB 8.4 MB/s eta 0:00:01
[K     |███████████▌                    | 40 kB 7.9 MB/s eta 0:00:01
[K     |██████████████▍                 | 51 kB 5.3 MB/s eta 0:00:01
[K     |█████████████████▏              | 61 kB 5.5 MB/s eta 0:00:01
[K     |████████████████████            | 71 kB 5.3 MB/s eta 0:00:01
[K     |███████████████████████         | 81 kB 5.9 MB/s eta 0:00:01
[K     |█████████████████████████▉      | 92 kB 6.1 MB/s eta 0:00:01
[K     |████████████████████████████▊   | 102 kB 5.3 MB/s eta 0:00:01
[K     |███████████████████████████████▋| 112 kB 5.3 MB/s eta 0:00:01
[K     |████████████████████████████████| 114 kB 5.3 MB/s 
[?25hInstalling collected packages: neattext
Successfully installed neattext-0.1.2<jupyter_text>Importing Libraries<jupyter_code>import transformers
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch
import requests
from bs4 import BeautifulSoup
import re
import seaborn as sns
import neattext.functions as nfx
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from wordcloud import WordCloud
from sklearn.feature_extraction.text import TfidfVectorizer
pd.set_option('display.max_colwidth', 1000) 
import nltk
from sklearn.feature_extraction.text import CountVectorizer
nltk.download('punkt')
nltk.download('stopwords')

from google_play_scraper.features.reviews import Sort, reviews_all, reviews
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation<jupyter_output>[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!<jupyter_text>Scrapping App Reviews from Google Play Store<jupyter_code>result,continuation_token= reviews('com.ubercab',lang='en',country='us',sort=Sort.MOST_RELEVANT, count=10000)
df = pd.DataFrame(result)
df
df_app=pd.DataFrame(np.array(df['content']),columns=['review'])
df_app<jupyter_output><empty_output><jupyter_text>Processing the Data<jupyter_code>df_app['review']=df_app['review'].apply(nfx.remove_hashtags).apply(nfx.remove_emojis).apply(nfx.remove_urls).apply(nfx.remove_userhandles).apply(nfx.remove_multiple_spaces).apply(nfx.remove_punctuations)<jupyter_output><empty_output><jupyter_text>Negative Reviews Ectraction(Sentiment Analysis) with BERT<jupyter_code>tokenizer = AutoTokenizer.from_pretrained('nlptown/bert-base-multilingual-uncased-sentiment')
model = AutoModelForSequenceClassification.from_pretrained('nlptown/bert-base-multilingual-uncased-sentiment')
def get_sentiment_score(text):
  tokens = tokenizer.encode(text, return_tensors='pt')
  result= model(tokens)
  return int(torch.argmax(result.logits))+1

df_app
#df_result.insert(1, "sentiment_scores", df_result['processed_text'].apply(lambda x: sentiment_score(x[:512])), True)
df_app['sentiment']=df_app['review'].apply(lambda x: get_sentiment_score(x[:512]))
negative_reviews = df_app[df_app['sentiment']<=3]['review']
negative_reviews
df_app=df_app.drop_duplicates()
df_negative=pd.DataFrame(np.array(negative_reviews),columns=['processed_review'])<jupyter_output><empty_output><jupyter_text>Topic Modelling with LDA<jupyter_code>cv = CountVectorizer(max_df=0.95, min_df=2, stop_words='english')
vectorizer = cv.fit_transform(df_negative['processed_review'])
LDA = LatentDirichletAllocation(n_components=3,random_state=1)
LDA.fit(vectorizer)
for index,topic in enumerate(LDA.components_):
    print(f'topic #{index} : ')
    print([cv.get_feature_names()[i] for i in topic.argsort()[-20:]])
topic_results = LDA.transform(vectorizer)
df_topic_results = pd.DataFrame(topic_results, columns=[
'0_Service/App related Issues',
'1_Payment Issues' ,
'2_Driver/Ride Issues'            ,
'3_Booking/Cancellation Issues'  , 
])
df_result = pd.merge(df_negative, df_topic_results,  how='inner', left_index=True, right_index=True )
df_result.to_csv('app_reviews_bms.csv')<jupyter_output><empty_output><jupyter_text>Final Topics-Reviews Visualization<jupyter_code>df_visualization= pd.read_csv("app_reviews_bms.csv")
df_visualization
df_visualization['0_Service/App related Issues']=round(df_visualization['0_Service/App related Issues']*100,2)
df_visualization['1_Payment Issues']=round(df_visualization['1_Payment Issues']*100,2)
df_visualization['2_Driver/Ride Issues']=round(df_visualization['2_Driver/Ride Issues']*100,2)
df_visualization['3_Booking/Cancellation Issues']=round(df_visualization['3_Booking/Cancellation Issues']*100,2)
df_visualization['0_Service/App related Issues']=df_visualization['0_Service/App related Issues'].astype(str)+"%"
df_visualization['1_Payment Issues']=df_visualization['1_Payment Issues'].astype(str)+"%"
df_visualization['2_Driver/Ride Issues']=df_visualization['2_Driver/Ride Issues'].astype(str)+"%"
df_visualization['3_Booking/Cancellation Issues']=df_visualization['3_Booking/Cancellation Issues'].astype(str)+"%"

df_visualization
df_list=df_visualization.values.tolist()
df_list
counter0=0
counter1=0
counter2=0
counter3=0

for i in range(0,len(df_list)):
  temp=[float(df_list[i][2][:-1]),float(df_list[i][3][:-1]),float(df_list[i][4][:-1]),float(df_list[i][5][:-1])]
  if(i==0):
    print(temp)
  if(max(temp)==float(df_list[i][2][:-1])):
    counter0+=1
  elif(max(temp)==float(df_list[i][3][:-1])):
    counter1+=1
  elif(max(temp)==float(df_list[i][4][:-1])):
    counter2+=1
  elif(max(temp)==float(df_list[i][5][:-1])):
    counter3+=1
  
print(len(df_list))
print(counter0)
print(counter1)
print(counter2)
print(counter3)
# Pie chart
labels = ['Service/App related Issues',
'Payment Issues' ,
'Driver/Ride Issues'            ,
'Booking/Cancellation Issues']
sizes = [counter0, counter1, counter2, counter3]
# only "explode" the 2nd slice (i.e. 'Hogs')
explode = (0, 0.1, 0, 0)  
fig1, ax1 = plt.subplots()
ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
        shadow=True, startangle=90)
# Equal aspect ratio ensures that pie is drawn as a circle
ax1.axis('equal')  
plt.tight_layout()
plt.show()
import matplotlib.pyplot as plt
# Pie chart
labels = ['Service/App related Issues',
'Payment Issues' ,
'Driver/Ride Issues'            ,
'Booking/Cancellation Issues']
sizes = [counter0, counter1, counter2, counter3]
#colors
colors = ['#FF0000','#00ABF0','#3CB043','#FFA500']
 
fig1, ax1 = plt.subplots()
ax1.pie(sizes, colors = colors, labels=labels, autopct='%1.1f%%', startangle=90)
#draw circle
centre_circle = plt.Circle((0,0),0.75,fc='white')
fig = plt.gcf()
fig.gca().add_artist(centre_circle)
# Equal aspect ratio ensures that pie is drawn as a circle
ax1.axis('equal')  
plt.tight_layout()
plt.show()
plt.barh(labels, sizes, color=colors)
plt.xlabel("Number of complaints",fontweight ='bold')
plt.ylabel("Topics for Negative Reviews",fontweight ='bold')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16727321741474593, 'fraction_numerical': 0.03679741205014153, 'mean_word_length': 6.267384916748286, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '138773', 'license': 'CC-BY-3.0', 'n_tokens_mistral': 3076, 'n_tokens_neox': 2751, 'n_words': 475, 'path': 'test.ipynb', 'repo_name': 'zeeshansalim1234/401_project_personal', 'size': '7419'}