<jupyter_start><jupyter_text>***Fast food reviews analysis*** For my analysis I used different datasets of fastfood and parsed them from Appstore Parsing of datasets<jupyter_code>!pip install app_store_scraper
from app_store_scraper import AppStore 
import pandas as pd 
import numpy as np
import json
mcd = AppStore(country="us", app_name="McDonald's") 
mcd.review(how_many=200)
df = pd.DataFrame(np.array(mcd.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['McDonalds']*200
df2.head()
df2.to_csv('/content/data/App Store Review McDonalds.csv')
pizza = AppStore(country="us", app_name="Wendy's") 
pizza.review(how_many= 200)
df = pd.DataFrame(np.array(pizza.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['Wendys']*200
df2.head()
df2.to_csv('/content/data/App Store Review Wendys.csv')
taco = AppStore(country="us", app_name="<NAME>") 
taco.review(how_many=200)

df = pd.DataFrame(np.array(taco.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['Taco Bell']*200
df2.head()
df2.to_csv('/content/data/App Store Review Taco Bell.csv')
bk = AppStore(country="us", app_name="KFC") 
bk.review(how_many=200)

df = pd.DataFrame(np.array(bk.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['KFC']*200
df2.head()
df2.to_csv('/content/data/App Store Review KFC.csv')
pap = AppStore(country="us", app_name="Domino's") 
pap.review(how_many=200)
df = pd.DataFrame(np.array(pap.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['Dominos']*200
df2.head()
df2.to_csv('/content/data/App Store Review Dominos.csv')
sub = AppStore(country="us", app_name="Subway") 
sub.review(how_many=200)

df = pd.DataFrame(np.array(sub.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['Subway']*200
df2.head()
df2.to_csv('/content/data/App Store Review Subway.csv')
krispy = AppStore(country="us", app_name="<NAME>") 
krispy.review(how_many=200)
df = pd.DataFrame(np.array(krispy.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['<NAME>']*200
df2.head()
df2.to_csv('/content/data/App Store Review Krispy Kreame.csv')
baskin = AppStore(country="us", app_name="Baskin-Robbins") 
baskin.review(how_many=200)
df = pd.DataFrame(np.array(baskin.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['<NAME>']*200
df2.head()
df2.to_csv('/content/data/App Store Review Baskin Robbins.csv')
fri = AppStore(country="us", app_name="Panda Express") 
fri.review(how_many=200)
df = pd.DataFrame(np.array(fri.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['Panda Express']*200
df2.head()

df2.to_csv('/content/data/App Store Review Panda Express.csv')
fri = AppStore(country="us", app_name="Boston Market") 
fri.review(how_many=200)
df = pd.DataFrame(np.array(fri.reviews),columns=['review']) 
df2 = df.join(pd.DataFrame(df.pop('review').tolist())) 
df2['Company_name'] = ['Boston Market']*200
df2.head()
df2.to_csv('/content/data/App Store Review Boston Market.csv')
import pandas as pd    #  pip install pandas
from pathlib import Path

data_dir = Path("/content/data")

df = pd.concat([pd.read_csv(f) for f in data_dir.glob("*.csv")], ignore_index=True)
df.to_csv("/content/data/result.csv", index=False)
df = pd.read_csv('/content/data/result.csv')
df.head()
df.shape
df = df.drop('Unnamed: 0', axis=1)
df<jupyter_output><empty_output><jupyter_text>Wordcountwith Spark<jupyter_code>!pip install pyspark py4j
import sys
 
from operator import add
from pyspark.sql import SparkSession
from pyspark.ml.feature import Tokenizer
from pyspark.ml.feature import StopWordsRemover
import pyspark.sql.functions as f
from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate() 
spark
spark = SparkSession\
  .builder \
  .appName("WordCounter") \
  .getOrCreate()
data = spark.read.format('csv').options(header='true', inferSchema='true') \
  .load('/content/data/result.csv') \
 
print('############ CSV extract:')
data.show()
df = data.drop('Unnamed: 0','date','isEdited','userName','developerResponse','rating','Company_name','title')
df.show()
df = pd.read_csv('/content/data/result.csv')
df = df['review']
df
df.to_csv("test.txt", header=None, index=None, sep=',', mode='w')
spark=SparkSession.builder\
    .master("local[*]")\
    .appName("WordCount")\
    .getOrCreate()
sc=spark.sparkContext
fastfood="/content/test.txt"
fastfoodrdd=sc.textFile(fastfood)
fastfoodrdd.take(50)
def lower_clean_str(x):
  punc='!"#$%&\'()*+,./:;<=>?@[\\]^_`{|}~-'
  lowercased_str = x.lower()
  for ch in punc:
    lowercased_str = lowercased_str.replace(ch, '')
  return lowercased_str
fastfoodrdd = fastfoodrdd.map(lower_clean_str)
fastfoodrdd.take(10)
fastfoodrdd=fastfoodrdd.flatMap(lambda word: word.split(" "))
fastfoodrdd.take(10)
fastfoodrdd = fastfoodrdd.filter(lambda x:x!='')
fastfoodrdd.take(20)
fastfoodrdd=fastfoodrdd.map(lambda  word:(word,1))
fastfoodrdd.take(7)
fastfoodrdd=fastfoodrdd.reduceByKey(lambda x,y:(x+y)).sortByKey()
fastfoodrdd.take(7)
fastfoodrdd=fastfoodrdd.map(lambda x:(x[1],x[0]))
fastfoodrdd.take(6)
fastfoodrdd.sortByKey(False).take(10)
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
stopwords =stopwords.words('english')
stopwords
fastfoodrdd = fastfoodrdd.filter(lambda x: x[1] not in stopwords).sortByKey(False)
fastfoodrdd.sortByKey(False).take(10)<jupyter_output><empty_output><jupyter_text>Analysis of dataset<jupyter_code>df = pd.read_csv('/content/data/result.csv')
df
#created aggregated information
import seaborn as sns
x = df.groupby(["Company_name"], as_index=False).agg({"rating": np.mean})
x
df['num_words_in_review']=df['review'].str.split().map(lambda x: len(x))
import matplotlib.pyplot as plt
import seaborn as sns
plt.style.use('fivethirtyeight')

# for advanced visualizations
import plotly.offline as py
from plotly.offline import init_notebook_mode, iplot
import plotly.graph_objs as go
from plotly import tools
init_notebook_mode(connected = True)
import plotly.figure_factory as ff

import warnings
warnings.filterwarnings('ignore')

plt.rcParams['figure.figsize'] = (12, 7)
plt.style.use('fivethirtyeight')

sns.boxplot(df['rating'], df['num_words_in_review'], palette = 'Blues')
plt.title("Length of review vs Ratings")
plt.show()
plt.rcParams['figure.figsize'] = (15, 9)
plt.style.use('fivethirtyeight')

sns.boxenplot(df['Company_name'], df['rating'], palette = 'spring')
plt.title("Company_names vs Ratings")
plt.xticks(rotation = 90)
plt.show()

plt.rcParams['figure.figsize'] = (15, 9)
plt.style.use('fivethirtyeight')

sns.swarmplot(df['Company_name'], df['num_words_in_review'], palette = 'deep')
plt.title("Companies vs Length of review")
plt.xticks(rotation = 90)
plt.show()
df
import re
features=df['review'].values
#=== 
processed_features = []

for sentence in range(0, len(features)):
    
    processed_feature = re.sub(r'\W', ' ', str(features[sentence]))
    # Remove all the special characters

    # Remove all single characters
    processed_feature= re.sub(r'\s+[a-zA-Z]\s+', ' ', processed_feature)

    # Remove single characters from the start
    processed_feature = re.sub(r'\^[a-zA-Z]\s+', ' ', processed_feature) 

    # Substituting multiple spaces with single space
    processed_feature = re.sub(r'\s+', ' ', processed_feature, flags=re.I)

    # Converting to Lowercase
    processed_feature = processed_feature.lower()

    processed_features.append(processed_feature)
df3=pd.DataFrame()
df3['features']=processed_features

#=======
from textblob import TextBlob
from wordcloud import WordCloud
# Create a function to get the subjectivity
def getSubjectivity(text):
   return TextBlob(text).sentiment.subjectivity

# Create a function to get the polarity
def getPolarity(text):
   return  TextBlob(text).sentiment.polarity


# Create two new columns 'Subjectivity' & 'Polarity'
df3['Subjectivity'] = df3['features'].apply(getSubjectivity)
df3['Polarity'] = df3['features'].apply(getPolarity)
#Create a function to compute negative (-1), neutral (0) and positive (+1) analysis
def getAnalysis(score):
 if score < 0:
  return 'Negative'
 elif score == 0:
  return 'Neutral'
 else:
  return 'Positive'
df3['Analysis'] = df3['Polarity'].apply(getAnalysis)
df3
Positive_sent = df3[df3['Analysis']=='positive']
Negative_sent = df3[df3['Analysis']=='negative']
Neutral_sent = df3[df3['Analysis']=='neutral']
!pip install nltk
 import nltk
 nltk.download('stopwords')
df3['temp_list'] = df3['features'].apply(lambda x:str(x).split())

#df3['temp_list'] = df3['temp_list'].astype('string')
#df3.dtypes
import nltk
nltk.download('punkt')
def remove_stopword(x):
    return [y for y in x if y not in stopwords.words('english')]
df3['temp_list'] = df3['temp_list'].apply(lambda x:remove_stopword(x))
df3
Positive_sent = df3[df3['Analysis']=='Positive']
Negative_sent = df3[df3['Analysis']=='Negative']
Neutral_sent = df3[df3['Analysis']=='Neutral']
df3['Analysis'].value_counts()
from collections import Counter
top = Counter([item for sublist in Positive_sent['temp_list'] for item in sublist])
temp_positive = pd.DataFrame(top.most_common(20))
temp_positive.columns = ['Common_words','count']
temp_positive.style.background_gradient(cmap='Greens')
top = Counter([item for sublist in Negative_sent['temp_list'] for item in sublist])
temp_negative = pd.DataFrame(top.most_common(20))
temp_negative = temp_negative.iloc[1:,:]
temp_negative.columns = ['Common_words','count']
temp_negative.style.background_gradient(cmap='Reds')
top = Counter([item for sublist in Neutral_sent['temp_list'] for item in sublist])
temp_neutral = pd.DataFrame(top.most_common(20))
temp_neutral = temp_neutral.loc[1:,:]
temp_neutral.columns = ['Common_words','count']
temp_neutral.style.background_gradient(cmap='Reds')
!pip install plotly-express
#numberofcharacters
def length(text):    
    '''a function which returns the length of text'''
    return len(text)
df['length'] = df['review'].apply(length)
import matplotlib.pyplot as plt
import seaborn as sns
sns.displot(df, x="length")
df['num_words_in_review']=df['review'].str.split().map(lambda x: len(x))
sns.displot(df, x='num_words_in_review')

import seaborn as sns
word=df['review'].str.split().apply(lambda x : [len(i) for i in x])
sns.distplot(word.map(lambda x: np.mean(x)),color='blue')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17159208180880006, 'fraction_numerical': 0.016887137630171686, 'mean_word_length': 6.741466957153231, 'pattern_counts': {'":': 1, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '401251', 'license': 'MIT', 'n_tokens_mistral': 4024, 'n_tokens_neox': 3846, 'n_words': 718, 'path': 'rakhimkulova_aliia_fastfood_analysis_.ipynb', 'repo_name': 'aliarahimckulova/fastfood_wordcount_rakhimkulova', 'size': '10682'}