<jupyter_start><jupyter_text>1. Problem statement: - In this project, Natural Language Processing (NLP) strategies will be used to analyz}e Yelp reviews data- Number of 'stars' indicate the business rating given by a customer, ranging from 1 to 5  2. Import library<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import string
import nltk
from nltk.corpus import stopwords

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>3. EDA<jupyter_code>yelp_df = pd.read_csv('yelp.csv')
yelp_df.head()
yelp_df.describe()<jupyter_output><empty_output><jupyter_text>3.1 Data quality check<jupyter_code>yelp_df.info()
yelp_df['text'].iloc[0]
yelp_df['text'].iloc[99]<jupyter_output><empty_output><jupyter_text>3.2 Review length histogram<jupyter_code>yelp_df['length'] = yelp_df['text'].apply(len)
yelp_df.head()
yelp_df['length'].plot(bins = 100, kind = 'hist')
yelp_df.length.describe()<jupyter_output><empty_output><jupyter_text>find the longest review test<jupyter_code>yelp_df[ yelp_df['length'] == 5039 ]['text'].iloc[0]
yelp_df[ yelp_df['length'] == 1 ]['text'].iloc[0]
yelp_df[ yelp_df['length'] == 710 ]['text'].iloc[0]<jupyter_output><empty_output><jupyter_text>3.3 Star count plot<jupyter_code>sns.countplot( y = 'stars', data = yelp_df)<jupyter_output><empty_output><jupyter_text>3.4 Histogram of review length per star<jupyter_code>g = sns.FacetGrid(data = yelp_df, col = 'stars', col_wrap = 3)
g.map(plt.hist, 'length', bins = 20, color = 'r')<jupyter_output><empty_output><jupyter_text>3.5 Visualize 1 star and 5 star review<jupyter_code>yelp_df_1 = yelp_df[ yelp_df['stars'] == 1 ]
yelp_df_5 = yelp_df[ yelp_df['stars'] == 5 ]
yelp_df_1_5 = pd.concat([yelp_df_1, yelp_df_5])
yelp_df_1_5.info()
print('1 start percentage = ', len(yelp_df_1)/ len(yelp_df_1_5) * 100, '%')
print('5 start percentage = ', len(yelp_df_5)/ len(yelp_df_1_5) * 100, '%')
sns.countplot(yelp_df_1_5['stars'], label = 'Count')<jupyter_output><empty_output><jupyter_text>4. sklearn CountVectorizer demo<jupyter_code># sample_data = ['hello world', 'hello world world hello', 'hello hello you world']

# from sklearn.feature_extraction.text import CountVectorizer

# vectorizer = CountVectorizer()

# X = vectorizer.fit_transform(sample_data)
# print(vectorizer.get_feature_names())
# print(X.toarray())<jupyter_output><empty_output><jupyter_text>5. Process data  5.1 clean data(stopword and punc)<jupyter_code>#performs the following: (1) remove punctuation, (2) remove stopwords
def message_cleaning(message):
    Test_punc_removed = [char for char in message if char not in string.punctuation]
    Test_punc_removed_join = ''.join(Test_punc_removed)
    Test_punc_removed_join_clean = [word for word in Test_punc_removed_join.split() if word.lower() not in stopwords.words('english')]
    return Test_punc_removed_join_clean
# yelp_df_clean = yelp_df_1_5['text'].apply(text_cleaning)
# yelp_df_clean
# yelp_df_clean[0]
# yelp_df_1_5['text'][0]<jupyter_output><empty_output><jupyter_text>5.2 Vectorize data<jupyter_code># Define the cleaning pipeline we defined earlier
vectorizer = CountVectorizer(analyzer = message_cleaning)
yelp_countvectorizer = vectorizer.fit_transform(yelp_df_1_5['text'])
yelp_countvectorizer.toarray()
print(vectorizer.get_feature_names())
yelp_countvectorizer.shape<jupyter_output><empty_output><jupyter_text>6. Train model<jupyter_code>X = yelp_countvectorizer
y =  yelp_df_1_5['stars']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
print(X_train.shape, y_train.shape)
NB_classifier = MultinomialNB()

NB_classifier.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>7. Evaluate model<jupyter_code>y_test_pred = NB_classifier.predict(X_test)
cm = confusion_matrix(y_test, y_test_pred)
sns.heatmap(cm, annot = True)
print(classification_report(y_test, y_test_pred))<jupyter_output>precision    recall  f1-score   support

           1       0.81      0.72      0.77       145
           5       0.94      0.96      0.95       673

    accuracy                           0.92       818
   macro avg       0.88      0.84      0.86       818
weighted avg       0.92      0.92      0.92       818<jupyter_text>8.  ID-IDF for feature extraction - TF: term frequency is used to measure the frequency of a term in a document; number of a word in a document / total word count;  - IDF: Inverse document frequency: log_e(total number of docs / number of docs with the word)<jupyter_code>yelp_tfidf = TfidfTransformer().fit_transform(yelp_countvectorizer)

print(yelp_tfidf.shape)
X = yelp_tfidf
y = yelp_df_1_5['stars']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)


NB_classifier = MultinomialNB()

NB_classifier.fit(X_train, y_train)

from sklearn.metrics import classification_report, confusion_matrix
y_test_pred = NB_classifier.predict(X_test)
cm = confusion_matrix(y_test, y_test_pred)
sns.heatmap(cm, annot = True)
print(classification_report(y_test, y_test_pred))<jupyter_output>precision    recall  f1-score   support

           1       1.00      0.01      0.01       165
           5       0.80      1.00      0.89       653

    accuracy                           0.80       818
   macro avg       0.90      0.50      0.45       818
weighted avg       0.84      0.80      0.71       818
{'fraction_non_alphanumeric': 0.10811775200713648, 'fraction_numerical': 0.03639607493309545, 'mean_word_length': 4.882476390346275, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '482205', 'license': 'MIT', 'n_tokens_mistral': 2231, 'n_tokens_neox': 2063, 'n_words': 425, 'path': 'yelp_review/yelp_review_rating_prediction.ipynb', 'repo_name': 'luke4u/Review_classification', 'size': '5605'}