<jupyter_start><jupyter_text>Validation<jupyter_code>import lightgbm as lgbm

X_train = proc_train.drop('sid', axis=1).reset_index(drop=True)
y_train = trn_session.target.reset_index(drop=True)

X_val = proc_val.drop('sid', axis=1).reset_index(drop=True)
y_val = val_session.target.reset_index(drop=True)

X_test = proc_test.drop('sid', axis=1).reset_index(drop=True)

temp_columns = X_train.columns
X_val = X_val[temp_columns]
X_test = X_test[temp_columns]
from sklearn.decomposition import PCA

pca = PCA(n_components = 10).fit(X_train)

X_train_pca = pd.DataFrame(pca.transform(X_train))
X_val_pca = pd.DataFrame(pca.transform(X_val))

X_train = pd.concat([X_train, X_train_pca], axis=1)
X_val = pd.concat([X_val, X_val_pca], axis=1)
# parameters for LightGBMClassifier
params = {
    'objective' :'binary',
    'max_depth' : 7,
    'boosting_type' : 'dart',
    'xgboost_dart_mode': True,
    'learning_rate': 0.1,
    'metric': 'binary_logloss',
    'verbose': 1
    #'bagging_fraction': 0.5,
}

d_train = lgbm.Dataset(X_train, label=trn_session.target)
d_valid = lgbm.Dataset(X_val, label=val_session.target)

clf = lgbm.train(params, d_train, valid_sets=[d_valid],
                num_boost_round=100)<jupyter_output>[1]	valid_0's binary_logloss: 0.308859
[2]	valid_0's binary_logloss: 0.302183
[3]	valid_0's binary_logloss: 0.29728
[4]	valid_0's binary_logloss: 0.29347
[5]	valid_0's binary_logloss: 0.290558
[6]	valid_0's binary_logloss: 0.288123
[7]	valid_0's binary_logloss: 0.286237
[8]	valid_0's binary_logloss: 0.285469
[9]	valid_0's binary_logloss: 0.284094
[10]	valid_0's binary_logloss: 0.282928
[11]	valid_0's binary_logloss: 0.28211
[12]	valid_0's binary_logloss: 0.281856
[13]	valid_0's binary_logloss: 0.281183
[14]	valid_0's binary_logloss: 0.280541
[15]	valid_0's binary_logloss: 0.279941
[16]	valid_0's binary_logloss: 0.279568
[17]	valid_0's binary_logloss: 0.27917
[18]	valid_0's binary_logloss: 0.278809
[19]	valid_0's binary_logloss: 0.278536
[20]	valid_0's binary_logloss: 0.278366
[21]	valid_0's binary_logloss: 0.278304
[22]	valid_0's binary_logloss: 0.278174
[23]	valid_0's binary_logloss: 0.278009
[24]	valid_0's binary_logloss: 0.277905
[25]	valid_0's binary_logloss: 0.27778
[26]	valid_0's[...]<jupyter_text>Model<jupyter_code>X_all = proc_all.drop('sid', axis=1)
y_all = all_session.target

X_test = proc_test.drop('sid', axis=1)

temp_columns = X_all.columns
X_test = X_test[temp_columns]

from sklearn.decomposition import PCA

pca = PCA(n_components = 10).fit(X_all)
X_all = pd.concat([X_all, pd.DataFrame(pca.transform(X_all))], axis=1)
X_test = pd.concat([X_test, pd.DataFrame(pca.transform(X_test))], axis=1)

import lightgbm as lgbm

# parameters for LightGBMClassifier
params = {
    'objective' :'binary',
    'max_depth' : 7,
    'boosting_type' : 'dart',
    'xgboost_dart_mode': True,
    'learning_rate': 0.1,
    #'metric': 'binary_logloss',
    #'bagging_fraction': 0.5,
}

d_train = lgbm.Dataset(X_all, label=y_all)
#d_valid = lgbm.Dataset(X_val, label=y_val)

clf = lgbm.train(params, d_train,# valid_sets=[d_valid],
                num_boost_round=100)
from matplotlib import rcParams
%matplotlib inline 

rcParams['figure.figsize'] = 10, 10
lgbm.plot_importance(clf)
count = 1
preds=clf.predict(X_test)
test_session['target'] = preds
test_session.to_csv(f'data/submissions/chu_{count}_submit.csv', index=False)
from sklearn.metrics import log_loss
count = 2
X_all.head()
y_all.head()
from sklearn.linear_model import RidgeCV
regr_cv = RidgeCV([5.5, 5.6, 5.7, 5.8, 6.0])
model_cv = regr_cv.fit(X_train, y_train)
cv_prediction = model_cv.predict(X_val)
cv_prediction
log_loss(y_val, cv_prediction)
len(X_all)
from scipy.stats import skew
from scipy.special import boxcox1p
numeric_feats = X_all.dtypes[X_all.dtypes != "object"].index
skewed_feats = X_train[numeric_feats].apply(lambda x: skew(x.dropna())) #compute skewness

skewed_feats = skewed_feats[skewed_feats > 0.65]

skewed_feats = skewed_feats.index
X_train[skewed_feats] = boxcox1p(X_train[skewed_feats], 0.15)
X_val[skewed_feats] = boxcox1p(X_val[skewed_feats], 0.15)
regr_cv = RidgeCV([5.5, 5.6, 5.7, 5.8, 6.0])
X_train = X_train.fillna(X_all.mean())
X_val = X_val.fillna(X_all.mean())
regr_cv.fit(X_train, y_train)
cv_prediction = model_cv.predict(X_val)
log_loss(y_val, cv_prediction)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13876702839990765, 'fraction_numerical': 0.07088432232740706, 'mean_word_length': 6.160330578512396, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '421749', 'license': 'MIT', 'n_tokens_mistral': 2073, 'n_tokens_neox': 1863, 'n_words': 321, 'path': 'notebooks/chu/2.0-linear_model.ipynb', 'repo_name': 'Robert-Alonso/DSG-2018-Finals', 'size': '4331'}