<jupyter_start><jupyter_text><jupyter_code>%matplotlib inline

import pandas as pd
import numpy as np

import seaborn as sns
import matplotlib.pyplot as plt

%config InlineBackend.figure_format = 'retina'
sns.set()
df = pd.read_csv('https://storage.googleapis.com/datascience360/booking_com_hotel_reviews.csv')
df.head()
df.tail()
df.info()
# Prepare the dataset, extract representative keywords from positive and negative reviews
from sklearn.feature_extraction.text import CountVectorizer

cv = CountVectorizer(analyzer = 'word', stop_words = 'english', max_features = 20,ngram_range=(2,2))
negative_features = cv.fit_transform(df['Negative_Review'])
negative_counts = negative_features.A.sum(axis=0)
negative_words = cv.get_feature_names()
negative_terms = dict(zip(negative_words,negative_counts))
top_negative_terms = sorted(negative_terms.items() ,  key=lambda x: x[1], reverse=True)
df_negative = pd.DataFrame(top_negative_terms, columns=["term", "frequency"])


positive_features = cv.fit_transform(df['Positive_Review'])
positive_counts = positive_features.A.sum(axis=0)
positive_words = cv.get_feature_names()
positive_terms = dict(zip(positive_words,positive_counts))
top_positive_terms = sorted(positive_terms.items() ,  key=lambda x: x[1], reverse=True)
df_positive = pd.DataFrame(top_positive_terms, columns=["term", "frequency"])
print('Top 20 terms in Negative Reviews:')
print('--------------------------------------------')
print(df_negative)

print()

print('Top 20 terms in Positive Reviews:')
print('--------------------------------------------')
print(df_positive)
sns.set(rc={'figure.figsize':(12,10)})
ax = sns.barplot(x="term", y="frequency", data = df_negative)
loc, labels = plt.xticks()
ax.set_xticklabels(labels, rotation=90);
neg = {"reviews":df['Negative_Review'],"sentiment": len(df['Negative_Review'])*["negative"]}
pos = {"reviews":df['Positive_Review'],"sentiment": len(df['Positive_Review'])*["positive"]}

df_senti = pd.DataFrame(neg).append(pd.DataFrame(pos))
df_senti.head()
df_senti.tail()
import nltk
nltk.download('stopwords')

from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer (max_features=150, min_df=7,  stop_words=stopwords.words('english'))
processed_features = vectorizer.fit_transform(df_senti['reviews']).toarray()
X = processed_features
y = df_senti['sentiment']

from sklearn.linear_model import LogisticRegression

lr = LogisticRegression()

lr.fit(X, y)
comment = "The food here are awful"
processed_comment = vectorizer.transform([comment])
lr.predict(processed_comment)
from sklearn.metrics import accuracy_score

predictions = lr.predict(X)

accuracy_score(y,predictions)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1628247347237468, 'fraction_numerical': 0.009147457006952067, 'mean_word_length': 6.552486187845304, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '404433', 'license': 'MIT', 'n_tokens_mistral': 934, 'n_tokens_neox': 889, 'n_words': 169, 'path': 'Untitled4.ipynb', 'repo_name': 'alienmortar/GREAT2014', 'size': '2733'}