<jupyter_start><jupyter_text>Loading dependencies<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from string import punctuation
from sklearn.preprocessing import LabelEncoder
from nltk.corpus import stopwords
from nltk.stem.porter import *
from gensim.models.phrases import Phraser, Phrases
from keras.preprocessing.text import one_hot
from keras.preprocessing.sequence import pad_sequences
import re
import string
from nltk.tokenize import word_tokenize
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Normalizer
from sklearn.manifold import TSNE
from sklearn.cross_validation import train_test_split
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import SGDClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score, confusion_matrix
from sklearn.metrics import precision_recall_curve, auc, roc_curve,classification_report,average_precision_score, precision_recall_fscore_support
from sklearn import metrics
from sklearn import cross_validation
from keras.models import Sequential
from sklearn.preprocessing import LabelEncoder
from keras.models import Model
from keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding, Flatten
from keras.layers.normalization import BatchNormalization
from keras.optimizers import RMSprop
from keras.preprocessing.text import Tokenizer
from keras.preprocessing import sequence
from keras.utils import to_categorical
from keras.callbacks import EarlyStopping
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
%matplotlib inline
Data = pd.read_csv('data.csv')
Data.head()
f,ax=plt.subplots(1,2,figsize=(18,8))
Data['sentiment'].value_counts().plot.pie(explode=[0,0.1],autopct='%1.1f%%',ax=ax[0],shadow=True)
ax[0].set_title('sentiment')
ax[0].set_ylabel('')
sns.countplot('sentiment',data=Data,ax=ax[1])
ax[1].set_title('sentiment')
plt.show()<jupyter_output><empty_output><jupyter_text>Cleaning Texts<jupyter_code>Data.content = Data.content.apply(lambda x: x.lower())
Data.content = Data.content.apply(lambda x : " ".join(re.findall('[\w]+',x)))
stop_words= stopwords.words('english')
def remove_stopWords(text):
    text = ' '.join(word for word in text.split() if word not in stop_words)
    return text

Data.content = Data.content.apply(lambda x: remove_stopWords(x))
Data.head()
# encoding the target variable using labelencoder
y = Data['sentiment']
LE = LabelEncoder()
y_LE = LE.fit_transform(y)
X = Data['content'].values.copy()
len(X), len(y)<jupyter_output><empty_output><jupyter_text>Changing to vectors<jupyter_code>X_counts = CountVectorizer().fit_transform(X)
X_counts.shape
X_counts_tdidf = TfidfTransformer(use_idf = True).fit_transform(X_counts)
X_counts_tdidf.shape<jupyter_output><empty_output><jupyter_text>Modeling  Splitting into train and test<jupyter_code>x_train, x_test, y_train, y_test = train_test_split(X, y_LE, test_size = 0.3)<jupyter_output><empty_output><jupyter_text>1: Random Forest  Using the defualt parameters<jupyter_code># The class weight of the classifier is set to balanced
RF_model = Pipeline([('vect', CountVectorizer()), 
                      ('tfidf', TfidfTransformer()), 
                      ('clf', RandomForestClassifier(n_jobs=-1, 
                                                     #max_features='sqrt', 
                                                     oob_score = True,
                                                     class_weight='balanced',
                                                     random_state = 46))])
fit_RFmodel = RF_model.fit(x_train, y_train)
RF_predicted = fit_RFmodel.predict(x_test)
RF_conf_mat=confusion_matrix(y_test, RF_predicted)
print ("Accuracy : " + str(np.mean(RF_predicted == y_test)))<jupyter_output>/anaconda2/lib/python2.7/site-packages/sklearn/ensemble/forest.py:453: UserWarning: Some inputs do not have OOB scores. This probably means too few trees were used to compute any reliable oob estimates.
  warn("Some inputs do not have OOB scores. "
/anaconda2/lib/python2.7/site-packages/sklearn/ensemble/forest.py:458: RuntimeWarning: invalid value encountered in true_divide
  predictions[k].sum(axis=1)[:, np.newaxis])<jupyter_text>The accuracy value of<jupyter_code>ax= plt.subplot()
sns.heatmap(RF_conf_mat, annot=True,fmt="d", ax = ax);
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); 
ax.set_title('Confusion Matrix for Random Forest'); 
ax.xaxis.set_ticklabels(['1', '0']); ax.yaxis.set_ticklabels(['1', '0']);
print(metrics.classification_report(y_test, RF_predicted))<jupyter_output>precision    recall  f1-score   support

          0       0.93      1.00      0.96      8998
          1       0.85      0.25      0.39       902

avg / total       0.92      0.93      0.91      9900<jupyter_text>As you can see the overall accuracy is good, but recall and f1-score for the miniority class is very low.  Tuning Random Forest hyperparameters<jupyter_code>RF_parameters = {'clf__max_depth': [3,None],
                 'clf__min_samples_leaf': [2, 5, 10]}
RF_GridSearch = GridSearchCV(estimator=RF_model, param_grid=RF_parameters,cv= 5)
RF_GridSearch = RF_GridSearch.fit(x_train, y_train)
print(RF_GridSearch.best_score_)
RF_BestGrid = RF_GridSearch.best_params_
print(RF_BestGrid)
Best_RF_model = Pipeline([('vect', CountVectorizer()), 
                      ('tfidf', TfidfTransformer()), 
                      ('clf', RandomForestClassifier(n_jobs=-1, 
                                                     max_features='auto',
                                                     max_depth = None,
                                                     min_samples_leaf = 2,
                                                     min_samples_split = 2,
                                                     oob_score = True,
                                                     class_weight='balanced',
                                                     random_state = 46))])
Best_fit_RFmodel = RF_model.fit(x_train, y_train)
Best_RF_predicted = Best_fit_RFmodel.predict(x_test)
Best_RF_conf_mat=confusion_matrix(y_test, Best_RF_predicted)
print ("Accuracy : " + str(np.mean(Best_RF_predicted == y_test)))
ax= plt.subplot()
sns.heatmap(Best_RF_conf_mat, annot=True,fmt="d", ax = ax);
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); 
ax.set_title('Confusion Matrix for tuned Random Forest'); 
ax.xaxis.set_ticklabels(['1', '0']); ax.yaxis.set_ticklabels(['1', '0']);
print(metrics.classification_report(y_test, Best_RF_predicted))<jupyter_output>precision    recall  f1-score   support

          0       0.93      1.00      0.96      8998
          1       0.85      0.25      0.39       902

avg / total       0.92      0.93      0.91      9900<jupyter_text>The model doesn't improve after tuning the model<jupyter_code>FP_BestRF, TP_BestRF, TH_BestRF = roc_curve(y_test, Best_RF_predicted)
ROC_BestRF = auc(FP_BestRF, TP_BestRF)
Precision_BestRF, Recall_BestRF, th_BestRF = precision_recall_curve(y_test, Best_RF_predicted)
# Plot ROC curve
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(FP_BestRF, TP_BestRF, label='RF (area = %0.3f)' % ROC_BestRF)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC curves for tuned SGDClassifiers')
plt.legend(loc='best')
plt.show()<jupyter_output><empty_output><jupyter_text>2: SVM  Using default parameters<jupyter_code># Again the class weight parameter is set to balanced
SVM_model = Pipeline([('vect', CountVectorizer()), 
                      ('tfidf', TfidfTransformer()), 
                      ('clf', SGDClassifier(class_weight='balanced',random_state = 46))])
fit_SVMmodel = SVM_model.fit(x_train, y_train)
SVM_predicted = fit_SVMmodel.predict(x_test)
SVM_conf_mat=confusion_matrix(y_test, SVM_predicted)
print ("Accuracy : " + str(np.mean(SVM_predicted == y_test)))
ax= plt.subplot()
sns.heatmap(SVM_conf_mat, annot=True,fmt="d", ax = ax);
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); 
ax.set_title('Confusion Matrix for SGDClassifier'); 
ax.xaxis.set_ticklabels(['1', '0']); ax.yaxis.set_ticklabels(['1', '0']);
print(metrics.classification_report(y_test, SVM_predicted))<jupyter_output>precision    recall  f1-score   support

          0       0.99      0.96      0.97      8998
          1       0.70      0.88      0.78       902

avg / total       0.96      0.95      0.96      9900<jupyter_text>SVM model performs better than random forest in terms of accuracy, precision and f1-score  tuning SVM<jupyter_code>SVM_parameters = {
    'tfidf__use_idf': (True, False),
    'clf__alpha': (0.00001,0.0001, 0.001,0.01),
    'clf__penalty': ('l2', 'elasticnet')
}
SVM_pipeline = Pipeline([('vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', SGDClassifier()),
])
SVM_GridSearch = GridSearchCV(SVM_pipeline, SVM_parameters, cv=None)
SVM_GridSearch=SVM_GridSearch.fit(x_train, y_train)
print(SVM_GridSearch.best_score_)
print(SVM_GridSearch.best_params_)
Best_SVM_model = Pipeline([('vect', CountVectorizer()), 
                      ('tfidf', TfidfTransformer()), 
                      ('clf', SGDClassifier(penalty = 'elasticnet',
                                           alpha=0.00001,
                                           fit_intercept=True,
                                           class_weight='balanced',
                                            random_state = 46))])
Best_fit_SVMmodel = Best_SVM_model.fit(x_train, y_train)
Best_SVM_predicted = Best_fit_SVMmodel.predict(x_test)
Best_SVM_conf_mat=confusion_matrix(y_test, Best_SVM_predicted)
print ("Accuracy : " + str(np.mean(Best_SVM_predicted == y_test)))
ax= plt.subplot()
sns.heatmap(Best_SVM_conf_mat, annot=True,fmt="d", ax = ax);
ax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); 
ax.set_title('Confusion Matrix for tuned SGDClassifier'); 
ax.xaxis.set_ticklabels(['1', '0']); ax.yaxis.set_ticklabels(['1', '0']);
print(metrics.classification_report(y_test, Best_SVM_predicted))<jupyter_output>precision    recall  f1-score   support

          0       0.99      0.96      0.97      8998
          1       0.70      0.87      0.78       902

avg / total       0.96      0.95      0.96      9900<jupyter_text>After tuning accuracy, recall and f1-score shows improvment, however precision slightly decreased.<jupyter_code>FP_BestSVM, TP_BestSVM, TH_BestSVM = roc_curve(y_test, Best_SVM_predicted)
ROC_BestSVM = auc(FP_BestSVM, TP_BestSVM)
Precision_BestSVM, Recall_BestSVM, th_BestSVM = precision_recall_curve(y_test, Best_SVM_predicted)
# Plot ROC curve
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(FP_BestSVM, TP_BestSVM, label='SVM (area = %0.3f)' % ROC_BestSVM)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC curves for tuned SGDClassifiers')
plt.legend(loc='best')
plt.show()<jupyter_output><empty_output><jupyter_text>3: Deep Learning<jupyter_code>num_max = 1000
LE = LabelEncoder()
y_LE = LE.fit_transform(y)
X_tok = Tokenizer(num_words=num_max)
X_tok.fit_on_texts(X)
X_mat = X_tok.texts_to_matrix(X,mode='count')
# Deep learning architecture
model = Sequential()
model.add(Dense(64, activation='relu', input_shape=(num_max,)))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
# Compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# Fit the model
history = model.fit(X_mat, y_LE, validation_split=0.3, epochs=40, batch_size=100, verbose=0)
# summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1162127929069031, 'fraction_numerical': 0.02351171627612413, 'mean_word_length': 3.8719629772464326, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '196025', 'license': 'MIT', 'n_tokens_mistral': 4411, 'n_tokens_neox': 4163, 'n_words': 838, 'path': 'Wipro_Proj_Final.ipynb', 'repo_name': 'asmiexr999/jupyternotebook', 'size': '12632'}