<jupyter_start><jupyter_text>Load the data<jupyter_code>mydir = ('file:/home/ubuntu/review.json')
df1 = sqlContext.read.json(mydir)
mydir2 = ('file:/home/ubuntu/business.json')
df2 = sqlContext.read.json(mydir2)
df = df1.join(df2,(df1.business_id==df2.business_id)).drop(df2.business_id).drop(df1.stars)<jupyter_output><empty_output><jupyter_text>Filter for chinese cuisine<jupyter_code>asian=df.where(array_contains(df.categories,"Chinese"))
asian.registerTempTable("asian")
asian_sample = sqlContext.sql("select business_id,text,stars from asian")
asian.registerTempTable("asian_sample")
asian_text=asian_sample.select('business_id','text')<jupyter_output><empty_output><jupyter_text>Filter the reviews for Service related aspects<jupyter_code>words = [x.lower() for x in ['servic','time','price','lunch','friendli']]

from pyspark.sql.types import *

def intersect(row):
    # convert each word in lowercase
    row = [x.lower() for x in row.split()]
    return True if set(row).intersection(set(words)) else False<jupyter_output><empty_output><jupyter_text>Filtering sentences based on aspects and aggregate the sentences for each restaurant<jupyter_code>import nltk.data
sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
summarised=asian_text.rdd.map(lambda (business,text):(business,sent_detector.tokenize(text.strip())))\
.map(lambda (business,text):(business," ".join(filter(lambda sent: intersect(sent),text))))\
.reduceByKey(lambda x,y:x+y).toDF()\
.withColumnRenamed('_1','BusinessId').withColumnRenamed('_2','Review')
from textblob import TextBlob, Word, Blobber
from textblob.classifiers import NaiveBayesClassifier
from textblob.taggers import NLTKTagger
asian_sample_rdd=summarised.rdd
asian_sample_polarity = asian_sample_rdd.map(lambda x:(x[0],TextBlob(x[1]).sentiment.polarity))
asian_pol1=asian_sample_polarity.toDF().withColumnRenamed('_1','business_id').withColumnRenamed('_2','pol1')<jupyter_output><empty_output><jupyter_text>Filter the reviews for Food related aspects<jupyter_code>words = [x.lower() for x in ['noodl','dish','chicken','fri','rice','soup','sauc','beef','pork','tast']]
from pyspark.sql.types import *
def intersect(row):
    # convert each word in lowercase
    row = [x.lower() for x in row.split()]
    return True if set(row).intersection(set(words)) else False<jupyter_output><empty_output><jupyter_text>Filtering sentences based on aspects and aggregate the sentences for each restaurant<jupyter_code>import nltk.data
sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
summarised=asian_text.rdd.map(lambda (business,text):(business,sent_detector.tokenize(text.strip())))\
.map(lambda (business,text):(business," ".join(filter(lambda sent: intersect(sent),text))))\
.reduceByKey(lambda x,y:x+y).toDF()\
.withColumnRenamed('_1','BusinessId').withColumnRenamed('_2','Review')<jupyter_output><empty_output><jupyter_text>Sentiment analysis using TextBlob<jupyter_code>asian_sample_rdd=summarised.rdd
asian_sample_polarity = asian_sample_rdd.map(lambda x:(x[0],TextBlob(x[1]).sentiment.polarity))
asian_pol2=asian_sample_polarity.toDF().withColumnRenamed('_1','business_id').withColumnRenamed('_2','pol2')<jupyter_output><empty_output><jupyter_text>Merge the topics for each restaurant<jupyter_code>asian_rating = sqlContext.sql("select distinct business_id,stars from asian_sample")
temp = asian_rating.join(asian_pol1,(asian_rating.business_id==asian_pol1.business_id),'left').drop(asian_rating.business_id)
asian_final=temp.join(asian_pol2,(temp.business_id==asian_pol2.business_id),'outer').drop(asian_pol2.business_id)
asian_final=asian_final.withColumn('pol1',when(isnull(asian_final.pol1), 0).otherwise(asian_final.pol1))
asian_final=asian_final.withColumn('pol2',when(isnull(asian_final.pol2), 0).otherwise(asian_final.pol2))asian_final=asian_final.filter(asian_final.business_id.isNotNull())
asian_final.registerTempTable("asian_final1")
asian_final1=sqlContext.sql("select * from asian_final1 where not pol1 ==0 and not pol2==0")<jupyter_output><empty_output><jupyter_text>Linear Regression Model<jupyter_code>from pyspark.ml.regression import LinearRegression
training = asian_final1
from pyspark.ml.feature import (VectorAssembler,VectorIndexer,
                                OneHotEncoder,StringIndexer)
numericCols = ['pol1','pol2']
assembler = VectorAssembler(inputCols=numericCols, outputCol="features")
lr = LinearRegression(maxIter=10, featuresCol="features",labelCol="stars")
training2=assembler.transform(training)
# Fit the model
lrModel = lr.fit(training2)<jupyter_output><empty_output><jupyter_text>Beta co-efficients<jupyter_code>lrModel.coefficients<jupyter_output><empty_output><jupyter_text>R-squared<jupyter_code>lrModel.summary.r2<jupyter_output><empty_output><jupyter_text>p-values<jupyter_code>lrModel.summary.pValues<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14038342609771182, 'fraction_numerical': 0.011131725417439703, 'mean_word_length': 10.231481481481481, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '115755', 'license': 'MIT', 'n_tokens_mistral': 1759, 'n_tokens_neox': 1678, 'n_words': 218, 'path': 'Sentiment_Analysis_Linear_Regression.ipynb', 'repo_name': 'yoyochrist/Aspect-Based-Opinion-Mining-Using-Spark', 'size': '4851'}