<jupyter_start><jupyter_text>File name: Investigation_3.ipynb               Author: <NAME>                      Latest Update: 26.07.2019<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import json
import nltk
from nltk.tokenize import RegexpTokenizer
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from collections import Counter
from sklearn.model_selection import train_test_split
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras import Sequential
from keras.layers import Embedding, Dense, LSTM, LeakyReLU
from keras.models import load_model<jupyter_output>Using TensorFlow backend.<jupyter_text>0) Task **Questions:**Each business in the YELP dataset is rated by the YELP community by reviews. By understanding the reviews and there positive/negative judgment, business can improve and optimize better to the wishes and needs of their customers.**Approach:**Step 1: Preprocessing the reviews and relabel the 'star' rating into positive and negative reviews. Step 2: Build up a Deep Learning Model for the sentiment analysis   1) Preprocessing  1.1) Load the data<jupyter_code>path_dataset_review = '../data/yelp_dataset/review.json'
# The 6.685.899 reviews of the dataset are to much for computation on a laptop. 
# Hence load just a view reviews manually.

# Define the number of reviews to load from the file
nr_reviews = 500000

# Load the reviews from the reviews dataset
file = open(path_dataset_review, encoding="utf8") 
all_data = list()
for i, line in enumerate(file):
    if i%(nr_reviews/10)==0:
        print(str(i*100/nr_reviews)+'% loaded' )
    if i == nr_reviews:
        break    
    data = json.loads(line)
    # extract only the review and the star rating form the line
    text = data['text']
    stars = data['stars']
    all_data.append([stars, text])
file.close()

# Create the DataFrame
df = pd.DataFrame(all_data, columns=['stars','text'])
df.head()<jupyter_output><empty_output><jupyter_text>1.2) EDA<jupyter_code># Distribution of the star ratings
star_count = df['stars'].value_counts()
star_count.reindex([1, 2, 3, 4, 5]).plot.bar()
df['text'][0]
df['text'][1]<jupyter_output><empty_output><jupyter_text>1.2.1) Increase the review quality<jupyter_code># Lowercase all words in the reviews
df['processed_text'] = df['text'].str.lower()
# Tokenize the reviews
# Simultaneously the punctation is removed
tokenize = RegexpTokenizer('\w+')
df['processed_text'] = df['processed_text'].apply(lambda review : tokenize.tokenize(review))
# Remove stopwords for the reviews
stoplist = stopwords.words('english')
df['processed_text'] = df['processed_text'].apply(lambda review: ' '.join([word for word in review if word not in stoplist]))
df['processed_text'].head()
# Stemming the words in the reviews
stemmer = SnowballStemmer('english')
df['processed_text'] = df['processed_text'].apply(lambda review: ' '.join([stemmer.stem(word) for word in review.split()]))
print('The max length of words in a review befor the preprocessing:', max(df['text'].agg(len)))
print('The max length of words in a review befor the preprocessing:', max(df['processed_text'].agg(len)))<jupyter_output>The max length of words in a review befor the preprocessing: 5000
The max length of words in a review befor the preprocessing: 3789<jupyter_text>1.3) Analyse the reviews<jupyter_code># 10 most common words
counter_words = Counter(" ".join(df['processed_text']).split())
print('Most common words:')
counter_words.most_common(10)
# 10 least common words
print('Least common words:')
counter_words.most_common()[-10:]
print('The max length of words in a review befor the preprocessing:', max(df['text'].agg(len)))
print('The max length of words in a review after the preprocessing:', max(df['processed_text'].agg(len)))
# Total different words
print('Number of different words in the review:', len(counter_words.most_common()))
# The last 100000 words are appearing very rearly
counter_words.most_common()[-100000]
counter_words.most_common()[25000]<jupyter_output><empty_output><jupyter_text>**Conclusion:** Only use the 25.000 most common words for this investigation<jupyter_code>VOCAB_LEN = 25000<jupyter_output><empty_output><jupyter_text>1.4) Maximal length of the reviews<jupyter_code># Distribution of the number of words in a review
reviews_len = [len(x) for x in df['processed_text']]
pd.Series(reviews_len).hist()
plt.title('Distribution of words')
plt.show()
pd.Series(reviews_len).describe()<jupyter_output><empty_output><jupyter_text>The histogram is showing, that the some reviews are written with 1000 and more words. For simplicity and reduction of complexity drop all reviews lower than the 10% quantile and greater than the 75% quantile.<jupyter_code>lower_bountary = np.quantile(reviews_len, 0.10)
upper_bountary = np.quantile(reviews_len, 0.75)
index = np.argwhere((reviews_len > lower_bountary) & (reviews_len < upper_bountary))
# Delete to short and to long reviews
df_reduce = df.loc[index.reshape(-1)]
df_reduce = df_reduce.reset_index(drop=True)
print('Shape of df:       ', df.shape)
print('Shape of df_reduce:', df_reduce.shape)<jupyter_output>Shape of df:        (500000, 3)
Shape of df_reduce: (323876, 3)<jupyter_text>1.5) Relabel the 'star' rating<jupyter_code># The value 1 indicates a positive review
# The value 0 indicates a negative review
df_reduce['stars_relabeled'] = [1 if x > 3 else 0 for x in df_reduce['stars']]
# Distribution of the relabeld star ratings
star_relabeld_count = df_reduce['stars_relabeled'].value_counts()
star_relabeld_count.reindex([0, 1]).plot.bar()<jupyter_output><empty_output><jupyter_text>1.6) Balance the dataset<jupyter_code># Oversampling the negative reviews
df_stars_1 = df_reduce[df_reduce['stars_relabeled']==1]
df_stars_0 = df_reduce[df_reduce['stars_relabeled']==0].sample(star_relabeld_count[1], replace=True, random_state=42)
df_balanced = pd.concat([df_stars_0, df_stars_1], axis=0)
df_balanced = df_balanced.reset_index(drop=True)
# Distribution of the relabeld star ratings
star_relabeld_balanced_count = df_balanced['stars_relabeled'].value_counts()
star_relabeld_balanced_count.reindex([0, 1]).plot.bar()<jupyter_output><empty_output><jupyter_text>1.7) Padding and pruning the reviews  For inputing the reviews in a neural network, all reviews have to be from equal length. Hence, pad to short reviews with 0 and prune to long reviews to the maximal length. At the same time the words are transformed to numbers.<jupyter_code>SEQ_LEN = 100
tokenizer = Tokenizer(num_words=VOCAB_LEN)
tokenizer.fit_on_texts(df_balanced['processed_text'])
sequences = tokenizer.texts_to_sequences(df_balanced['processed_text'])
X = pad_sequences(sequences, maxlen=SEQ_LEN)
target = df_balanced['stars_relabeled']
X.shape<jupyter_output><empty_output><jupyter_text>2) Train-, Validation-, Test-Set<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X , target, test_size=0.7, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.50, random_state=42)
print('Shape of X_train:', X_train.shape)
print('Shape of X_val:  ', X_val.shape)
print('Shape of X_test: ', X_test.shape)<jupyter_output>Shape of X_train: (67047, 100)
Shape of X_val:   (67047, 100)
Shape of X_test:  (312888, 100)<jupyter_text>3) Building the model  3.1) Neural Network (model 1) Building up a Neural Network with an Embedding Layer and an LSTM Layer.<jupyter_code>EMB_DIM = 100
model1 = Sequential()

model1.add(Embedding(VOCAB_LEN, EMB_DIM, input_length=SEQ_LEN))
model1.add(LSTM(units=EMB_DIM, dropout=0.4, recurrent_dropout=0.4))
model1.add(Dense(1, activation='sigmoid'))
model1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
model1.summary()
result_model1 = model1.fit(X_train, y_train, epochs=3, validation_data=(X_val, y_val))
score_train_m1, accu_train_m1 = model1.evaluate(X_train, y_train)
score_val_m1, accu_val_m1 = model1.evaluate(X_val, y_val)
score_test_m1, accu_test_m1 = model1.evaluate(X_test, y_test)
print('The accuracy of the first neural network:')
print('Test Set:      ', accu_train_m1)
print('Validation Set:', accu_val_m1)
print('Train Set:     ', accu_test_m1)
# Save the model
filename = 'LSTM_model_1.h5'
model1.save('../Models/Investigation_3/'+filename)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11155993822026851, 'fraction_numerical': 0.02661280741356778, 'mean_word_length': 6.543010752688172, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '97931', 'license': 'MIT', 'n_tokens_mistral': 2985, 'n_tokens_neox': 2780, 'n_words': 777, 'path': 'Notebooks/Investigation_3.ipynb', 'repo_name': 'imor-de/YELP', 'size': '8425'}