<jupyter_start><jupyter_text>Use logistic regression to A/B test engineered features.- Benchmark Combined log loss: 0.05601320817782069- Uppercase Count Combined log loss: 0.05493923371472729<jupyter_code>import os
import logging
import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code>def load_data(path):
    full_path = os.path.join(os.path.realpath('..'), path)
    df = pd.read_csv(full_path, header=0, index_col=0)
    print("Dataset has {} rows, {} columns.".format(*df.shape))
    return df
df_train = load_data('data/processed/train.csv')
df_test = load_data('data/processed/test.csv')
df_train.head()<jupyter_output><empty_output><jupyter_text>Pre-processing<jupyter_code>df = df_train
from sklearn.model_selection import train_test_split

seed = 42
np.random.seed(seed)
test_size = 0.2
target = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
corpus = 'comment_text'

X = df.drop(target, axis=1)
y = df[target]


Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=test_size, random_state=seed)
%%time
from sklearn.feature_extraction.text import TfidfVectorizer 

vectorizer = TfidfVectorizer( max_features = 20000, ngram_range = ( 1, 3 ), sublinear_tf = True )
Xtrain_TFIDF = vectorizer.fit_transform(Xtrain[corpus])
Xtest_TFIDF = vectorizer.transform(Xtest[corpus])
%%time

Xsubmission_TFIDF = vectorizer.fit_transform(df_test[corpus])<jupyter_output>CPU times: user 1min 58s, sys: 10.5 s, total: 2min 9s
Wall time: 2min 28s<jupyter_text>Model train<jupyter_code>from scipy import sparse

# Concat engineered features
features = ['uppercase_count', 'bad_words', 'typos', 'length']

Xtrain_combined = sparse.hstack((Xtrain_TFIDF, sparse.csr_matrix(Xtrain[features].values)))
Xtest_combined = sparse.hstack((Xtest_TFIDF, sparse.csr_matrix(Xtest[features].values)))
Xsubmission = sparse.hstack((Xsubmission_TFIDF, sparse.csr_matrix(df_test[features].values)))
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import log_loss
from sklearn.multiclass import OneVsRestClassifier
%%time

#Tune
param_grid = {'estimator__C': [0.1, 1, 10] }

# Fit model
clf = OneVsRestClassifier(LogisticRegression(), n_jobs=-1)
clf_cv = GridSearchCV(clf, param_grid, cv=3, verbose=1)
clf_cv.fit(Xtrain_combined, ytrain)

# Evaluate
y_pred = clf_cv.predict_proba(Xtest_combined)
hold_out_preds = pd.DataFrame(y_pred, index=ytest.index, columns=target)

losses = []
for label in target:
    loss = log_loss(ytest[label], hold_out_preds[label])
    losses.append(loss)
    print("{} log loss is {} .".format(label, loss))
    
print("Combined log loss: {} .".format(np.mean(losses)))
print(clf_cv.best_params_)<jupyter_output>Fitting 3 folds for each of 3 candidates, totalling 9 fits<jupyter_text>Outputs<jupyter_code>def save_df(df, path):
    full_path = os.path.join(os.path.realpath('..'), path)
    df.to_csv(full_path, header=True, index=True)
    print('Dataframe ({}, {}) saved as csv.'.format(*df.shape))
# submissions
model_name = 'LR'
y_submission = clf_cv.predict_proba(Xsubmission)
submission = pd.DataFrame(y_submission, index=df_test.index, columns=target)
save_df(submission, 'data/submissions/'+ model_name + '.csv')<jupyter_output>Dataframe (153164, 6) saved as csv.
{'fraction_non_alphanumeric': 0.1278127812781278, 'fraction_numerical': 0.023702370237023703, 'mean_word_length': 6.826291079812207, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '586799', 'license': 'MIT', 'n_tokens_mistral': 1301, 'n_tokens_neox': 1219, 'n_words': 240, 'path': 'notebooks/17-jc-LR.ipynb', 'repo_name': 'joaeechew/toxic', 'size': '3333'}