<jupyter_start><jupyter_text>For dockers + ensembles:<jupyter_code>os.environ['JOBLIB_TEMP_FOLDER'] = '../data/tmp/'<jupyter_output><empty_output><jupyter_text>Data Import<jupyter_code>train = pd.read_csv('data/labeledTrainData.tsv', sep='\t')
print(train.shape)
train.head()
test = pd.read_csv('data/testData.tsv', sep='\t')
print(test.shape)<jupyter_output>(25000, 2)<jupyter_text>Vectorize Text<jupyter_code>count_vec = CountVectorizer(ngram_range=(1,2), 
                            min_df=5,
                            max_df=0.9,
                            strip_accents='unicode',
                            max_features=None)
tfidf_vec = TfidfVectorizer(ngram_range=(1,2), 
                            min_df=3,
                            #max_df=0.9,
                            strip_accents='unicode',
                            analyzer='word',
                            token_pattern=r'\w{1,}',
                            use_idf=1,
                            smooth_idf=1,
                            sublinear_tf=1,
                            max_features=None,
                            stop_words = 'english')
count_vec.fit(train["review"].fillna("").values)
X_count = count_vec.transform(train["review"].fillna("").values)
print(type(X_count))
print(X_count.shape)
tfidf_vec.fit(train["review"].fillna("").values)
X_tfidf = tfidf_vec.transform(train["review"].fillna("").values)
print(type(X_tfidf))
print(X_tfidf.shape)
y = train['sentiment'].as_matrix()
print(y.shape)<jupyter_output>(25000,)<jupyter_text>Create Topic Vector<jupyter_code>dictionary = corpora.Dictionary.load('models/topic_dict.dict')
lda = LdaMulticore.load('models/topic_lda_model')
text_row_count = train['review'].shape[0]
vector_size = lda.num_topics
train['review_topic_vector'] = train['review'].apply(lambda x: lda[dictionary.doc2bow(x.lower().split(" "))])
X_lda_matrix = np.reshape(np.concatenate(train['review_topic_vector']
                                         .apply(lambda x: sparse2full(x, vector_size))
                                         .as_matrix(), axis=0), (text_row_count,vector_size))
test['review_topic_vector'] = test['review'].apply(lambda x: lda[dictionary.doc2bow(x.lower().split(" "))])
X_lda_matrix_test = np.reshape(np.concatenate(test['review_topic_vector']
                                         .apply(lambda x: sparse2full(x, vector_size))
                                         .as_matrix(), axis=0), (test['review'].shape[0],vector_size))<jupyter_output><empty_output><jupyter_text>Concat Features<jupyter_code>X_count_concat = hstack((X_count, X_lda_matrix))
X_tfidf_concat = hstack((X_tfidf, X_lda_matrix))
X_count_train, X_count_val, y_count_train, y_count_val = train_test_split(X_count_concat, y, test_size=0.1, random_state=2481632)
X_tfidf_train, X_tfidf_val, y_tfidf_train, y_tfidf_val = train_test_split(X_tfidf, y, test_size=0.1, random_state=2481632)
X_count_test = count_vec.transform(test['review'].fillna("").values)
X_tfidf_test = tfidf_vec.transform(test['review'].fillna("").values)
X_count_test = hstack((X_count_test, X_lda_matrix_test))
X_tfidf_test = hstack((X_tfidf_test, X_lda_matrix_test))<jupyter_output><empty_output><jupyter_text>Linear Models<jupyter_code>lm_logit = LogisticRegression(penalty='l2',
                              dual=True,
                              tol=0.0001,
                              C=1.0,
                              fit_intercept=True,
                              intercept_scaling=1.0, 
                              class_weight=None,
                              random_state=42,
                              solver='liblinear',
                              max_iter=1000,
                              verbose=1,
                              n_jobs=-1)
lm_logit.fit(X_tfidf_train, y_tfidf_train)
y_val_hat = lm_logit.predict_proba(X_tfidf_val)[:,1]
print(accuracy_score(y_tfidf_val, y_val_hat > 0.5))
print(roc_auc_score(y_tfidf_val, y_val_hat))
print(confusion_matrix(y_tfidf_val, y_val_hat > 0.5))
print(classification_report(y_tfidf_val, y_val_hat > 0.5))
print("20 Fold CV Score: {}".format(np.mean(cross_validation.cross_val_score(lm_logit, X_tfidf_train, y_tfidf_train, cv=20, scoring='roc_auc'))))
lm_logit_tfidf = LogisticRegression(penalty='l2',
                              C=1.0,
                              class_weight=None,
                              random_state=42,
                              solver='liblinear',
                              max_iter=1000,
                              verbose=1,
                              n_jobs=-1)
lm_logit_tfidf.fit(X_tfidf_train, y_tfidf_train)
y_tfidf_val_hat = lm_logit_tfidf.predict(X_tfidf_val)
print(accuracy_score(y_tfidf_val, y_tfidf_val_hat))
print(confusion_matrix(y_tfidf_val, y_tfidf_val_hat))
print(classification_report(y_tfidf_val, y_tfidf_val_hat))
lm_ridge = RidgeClassifierCV(alphas=(0.1, 0.5, 1.0, 5.0, 10.0), 
                             cv=5,
                             class_weight=None)
lm_ridge.fit(X_count_train, y_count_train)
lm_ridge.alpha_
y_ridge_val_hat = lm_ridge.predict(X_count_val)
print(accuracy_score(y_count_val, y_ridge_val_hat))
print(confusion_matrix(y_count_val, y_ridge_val_hat))
print(classification_report(y_count_val, y_ridge_val_hat))
lm_ridge_single = RidgeClassifier(alpha=10.0)
nb = MultinomialNB(alpha=0.5)
nb.fit(X_count_train, y_count_train)
y_nb_val_hat = nb.predict(X_count_val)
print(accuracy_score(y_count_val, y_nb_val_hat))
print(confusion_matrix(y_count_val, y_nb_val_hat))
print(classification_report(y_count_val, y_nb_val_hat))<jupyter_output>0.8688
[[1111  136]
 [ 192 1061]]
             precision    recall  f1-score   support

          0       0.85      0.89      0.87      1247
          1       0.89      0.85      0.87      1253

avg / total       0.87      0.87      0.87      2500<jupyter_text>NB-SVM Model Relevant Paper: https://nlp.stanford.edu/pubs/sidaw12_simple_sentiment.pdf<jupyter_code>from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.utils.validation import check_X_y, check_is_fitted
from sklearn.linear_model import LogisticRegression
from scipy import sparse

class NbSvmClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, C=1.0, dual=False, n_jobs=1):
        self.C = C
        self.dual = dual
        self.n_jobs = n_jobs
        self.coef_ = None

    def predict(self, x):
        # Verify that model has been fit
        check_is_fitted(self, ['_r', '_clf'])
        return self._clf.predict(x.multiply(self._r))

    def predict_proba(self, x):
        # Verify that model has been fit
        check_is_fitted(self, ['_r', '_clf'])
        return self._clf.predict_proba(x.multiply(self._r))

    def fit(self, x, y):
        # Check that X and y have correct shape
        y = y.values
        x, y = check_X_y(x, y, accept_sparse=True)

        def pr(x, y_i, y):
            p = x[y==y_i].sum(0)
            return (p+1) / ((y==y_i).sum()+1)

        self._r = sparse.csr_matrix(np.log(pr(x,1,y) / pr(x,0,y)))
        x_nb = x.multiply(self._r)
        self._clf = LogisticRegression(C=self.C, dual=self.dual, n_jobs=self.n_jobs).fit(x_nb, y)
        self.coef_ = self._clf.coef_
        return self
m = NbSvmClassifier(C=4, dual=True)
m.fit(x_nb, y_count_train)
y_hat = m.predict(X_count_val)
print(accuracy_score(y_count_val, y_hat))
print(confusion_matrix(y_count_val, y_hat))
print(classification_report(y_count_val, y_hat))<jupyter_output>0.9172
[[1133  114]
 [  93 1160]]
             precision    recall  f1-score   support

          0       0.92      0.91      0.92      1247
          1       0.91      0.93      0.92      1253

avg / total       0.92      0.92      0.92      2500<jupyter_text>Ensemble Model<jupyter_code>rf = RandomForestClassifier(n_estimators=300, n_jobs=-1)
rf.fit(X_count_train, y_count_train)
print(accuracy_score(y_count_val, rf.predict(X_count_val)))
print(confusion_matrix(y_count_val, rf.predict(X_count_val)))
print(classification_report(y_count_val, rf.predict(X_count_val)))
gbc = GradientBoostingClassifier(n_estimators=200, verbose=1)
gbc.fit(X_count_train, y_count_train)
print(accuracy_score(y_count_val, gbc.predict(X_count_val)))
print(confusion_matrix(y_count_val, gbc.predict(X_count_val)))
print(classification_report(y_count_val, gbc.predict(X_count_val)))
vote_m = VotingClassifier([('lm_ridge_single',lm_ridge_single), ('rf',rf), ('gbc',gbc)],
                          voting='hard',
                          n_jobs=-1)
vote_m.fit(X_count_train, y_count_train)
print(accuracy_score(y_count_val, vote_m.predict(X_count_val)))
print(confusion_matrix(y_count_val, vote_m.predict(X_count_val)))
print(classification_report(y_count_val, vote_m.predict(X_count_val)))<jupyter_output>0.8824
[[1075  172]
 [ 122 1131]]
             precision    recall  f1-score   support

          0       0.90      0.86      0.88      1247
          1       0.87      0.90      0.88      1253

avg / total       0.88      0.88      0.88      2500
{'fraction_non_alphanumeric': 0.1178626971784048, 'fraction_numerical': 0.0344367918240391, 'mean_word_length': 2.7264072847682117, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '514098', 'license': 'MIT', 'n_tokens_mistral': 3434, 'n_tokens_neox': 3254, 'n_words': 430, 'path': '02_baseline_models.ipynb', 'repo_name': 'smfullman/text_utils', 'size': '9002'}