<jupyter_start><jupyter_text>Sentiment Analysis ---You are currently looking at **version 1.0** of this notebook.--- *Note: Some of the cells in this notebook are computationally expensive. To reduce runtime, this notebook is using a subset of the data.*  Import libaries<jupyter_code>import pandas as pd
import numpy as np

pd.set_option('display.notebook_repr_html', True)<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code>!ls data | head -5
# Read in the data
df = pd.read_csv('data/Amazon_Unlocked_Mobile.csv')
# Sample the data to speed up computation
# Comment out this line to match with lecture
df = df.sample(frac=0.3, random_state=10)
df.head()
# Drop missing values
df.dropna(inplace=True)

# Remove any 'neutral' ratings equal to 3
df = df[df['Rating'] != 3]

# Encode 4s and 5s as 1 (rated positively)
# Encode 1s and 2s as 0 (rated poorly)
df['Positively Rated'] = np.where(df['Rating'] > 3, 1, 0)
df.head(10)
# Most ratings are positive
df['Positively Rated'].mean()
from sklearn.model_selection import train_test_split

# Split data into training and test sets
X_train, X_test, y_train, y_test = train_test_split(df['Reviews'], 
                                                    df['Positively Rated'], 
                                                    random_state=0)
X_train.iloc[0], X_train.shape<jupyter_output><empty_output><jupyter_text>CountVectorizer<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer

# Fit the CountVectorizer to the training data
vect = CountVectorizer().fit(X_train)
len(vect.get_feature_names()), 'features step 2000:', vect.get_feature_names()[::2000]
# transform the documents in the training data to a document-term matrix
X_train_vectorized = vect.transform(X_train)
X_train_vectorized<jupyter_output><empty_output><jupyter_text>Logistic regression- classify reviews as positive or not<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score
# Train the model
model = LogisticRegression()
model.fit(X_train_vectorized, y_train)
# Predict the transformed test documents
predictions = model.predict(vect.transform(X_test))
print('AUC: ', roc_auc_score(y_test, predictions))
# Find the N smallest and largest coefficients
def extreme_coeffs(feature_names, sorted_coef_index, N=10):
    print('Smallest Coefs:\n{}\n'.format(feature_names[sorted_coef_index[:N]]))
    print('Largest Coefs: \n{}'.format(feature_names[sorted_coef_index[::-1][:N]]))
# get the feature names as numpy array
feature_names = np.array(vect.get_feature_names())

# Sort the coefficients from the model
sorted_coef_index = model.coef_[0].argsort()

extreme_coeffs(feature_names, sorted_coef_index, 5)<jupyter_output><empty_output><jupyter_text>Tf-idf<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
# Fit the TfidfVectorizer to the training data specifiying a minimum document frequency of 5
vect = TfidfVectorizer(min_df=5).fit(X_train)
len(vect.get_feature_names())
X_train_vectorized = vect.transform(X_train)
model = LogisticRegression().fit(X_train_vectorized, y_train)

predictions = model.predict(vect.transform(X_test))
print('AUC: ', roc_auc_score(y_test, predictions))
feature_names = np.array(vect.get_feature_names())
sorted_tfidf_index = X_train_vectorized.max(0).toarray()[0].argsort()
extreme_coeffs(feature_names, sorted_tfidf_index, 5)
sorted_coef_index = model.coef_[0].argsort()
extreme_coeffs(feature_names, sorted_coef_index, 5)
# These reviews are treated the same by our current model
print(model.predict(vect.transform(['not an issue, phone is working',
                                    'an issue, phone is not working'])))<jupyter_output><empty_output><jupyter_text>n-grams<jupyter_code># Fit the CountVectorizer to the training data specifiying a minimum 
# document frequency of 5 and extracting 1-grams and 2-grams
vect = CountVectorizer(min_df=5, ngram_range=(2, 3)).fit(X_train)
X_train_vectorized = vect.transform(X_train)
len(vect.get_feature_names()), vect.get_feature_names()[:10]
X_train_vectorized
model = LogisticRegression()
model.fit(X_train_vectorized, y_train)

predictions = model.predict(vect.transform(X_test))
print('AUC: ', roc_auc_score(y_test, predictions))
feature_names = np.array(vect.get_feature_names())

sorted_coef_index = model.coef_[0].argsort()

extreme_coeffs(feature_names, sorted_coef_index)
# These reviews are now correctly identified
X_new = vect.transform(['not an issue, phone is working', 'an issue, phone is not working', 'not great', 'absolute shit', 'not epic'])
print(model.predict(X_new))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10660336642209754, 'fraction_numerical': 0.0110056107034959, 'mean_word_length': 5.949025487256372, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '664987', 'license': 'MIT', 'n_tokens_mistral': 1582, 'n_tokens_neox': 1518, 'n_words': 370, 'path': 'nlp_sentiment_analysis_2.ipynb', 'repo_name': 'aixpact/NLP', 'size': '4634'}