<jupyter_start><jupyter_text>Imports<jupyter_code># Basic imports
import pandas as pd
import numpy as np

# Warnings
import warnings 
warnings.simplefilter("ignore")
# Plot
from IPython import display
import seaborn as sns

import matplotlib
import matplotlib.pylab as plt
from jupyterthemes import jtplot

jtplot.style('gruvboxd')
matplotlib.use('nbagg')<jupyter_output><empty_output><jupyter_text>Data Reading<jupyter_code># Data Reading
df_train = pd.read_csv('./data/train.csv', index_col='Id')
df_test = pd.read_csv('./data/test.csv', index_col='Id') 

# Split X_train, y_train
target = 'SalePrice'
features = df_test.columns

y_train = df_train[target]
df_train = df_train[features]

df_train.head()<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>from robusta.preprocessing.category import *
from robusta.preprocessing.numeric import *
from robusta.preprocessing.base import *
from robusta.pipeline import *

data_prep = make_pipeline(
    ColumnSelector(columns=features),
    FeatureUnion([
        ("numeric", make_pipeline(
            TypeSelector(np.number),
            Imputer(strategy="median"),
            GaussRank(),
        )),
        ("category", make_pipeline(
            TypeSelector("object"),
            Imputer(strategy="most_frequent"),
            LabelEncoder(),
            TypeConverter('category'),
        )),
    ])
)

X_train = data_prep.fit_transform(df_train)
X_test = data_prep.transform(df_test)

X_train.head()<jupyter_output><empty_output><jupyter_text>Fold Preparation<jupyter_code>from robusta.outliers import *

detector = IsolationForest()

encoder = FeatureUnion([
    ('category', make_pipeline(
        TypeSelector(['category', 'object']), 
        TargetEncoderCV(cv=4).set_params(encoder__smoothing=200.0),
        Imputer(strategy="mean"),
    )),
    ('numeric', make_pipeline(
        TypeSelector(np.number),
        Identity(),
    )),
])

fold_pipe = make_pipeline(encoder, detector)

F_train, _ = fold_pipe.fit_resample(X_train, y_train)
F_train<jupyter_output>/Users/pechyonkinb/anaconda/lib/python3.6/site-packages/sklearn/ensemble/iforest.py:417: DeprecationWarning: threshold_ attribute is deprecated in 0.20 and will be removed in 0.22.
  " be removed in 0.22.", DeprecationWarning)<jupyter_text>Model<jupyter_code>from lightgbm import LGBMRegressor

model = LGBMRegressor()
%%time
from sklearn.model_selection import cross_val_score, GridSearchCV
from sklearn.metrics import mean_squared_log_error, make_scorer

rmsle = lambda y_true, y_pred: np.sqrt(mean_squared_log_error(y_true, y_pred))
scoring = make_scorer(rmsle, greater_is_better=False)
cv = 5

rmsle_scores = lambda y_true, y_pred: -np.abs(np.log1p(y_true) - np.log1p(y_pred))
super_detector = lambda estimator: make_pipeline(
    SupervisedOutlierDetector(estimator, rmsle_scores, -0.2), 
    estimator)

trans_target = lambda estimator: TransformedTargetRegressor(estimator, np.log1p, np.expm1)
from robusta.crossval import crossval_score

crossval_score(model, cv, X_train, y_train, scoring='neg_mean_squared_error')
from robusta.optimizer import OptunaCV, RandomSearchCV

param_space = {
    'learning_rate': 0.1,
    # Once your learning rate is fixed, do not change it.

    'n_estimators': (100, 2000, 100),
    
    'max_depth': (3, 12, 1),
    'num_leaves': {15, 31, 63, 127, 255, 511, 1023, 2047, 4095},
    
    'bagging_fraction': (0.1, 0.9, 0.05),
    'feature_fraction': (0.1, 0.9, 0.05),
    
    #'lambda_l1': (1e-6, 1e6, 'log'),
    #'lambda_l2': (1e-6, 1e6, 'log'),
}

opt = OptunaCV(model, cv, 'neg_mean_squared_error', param_space, verbose=2, plot=True,
               warm_start=True, n_trials=200)

opt.fit(X_train, np.log(y_train))
opt.best_estimator_
param_space = {
    #'learning_rate': 0.1,
    # Once your learning rate is fixed, do not change it.

    #'n_estimators': (100, 2000, 100),
    
    #'max_depth': (3, 12, 1),
    #'num_leaves': {15, 31, 63, 127, 255, 511, 1023, 2047, 4095},
    
    'bagging_fraction': (0.1, 0.9, 0.05),
    'feature_fraction': (0.1, 0.9, 0.05),
    
    #'lambda_l1': (1e-6, 1e6, 'log'),
    #'lambda_l2': (1e-6, 1e6, 'log'),
}

opt.set_params(param_space=param_space, n_trials=None, timeout=30, verbose=2)
opt.fit(X_train, np.log(y_train))
opt.best_estimator_
from itertools import combinations

for k in range(4):
    for modules in combinations(['detector', 'trans', 'encoder'], k):
        estimator = model
        if 'encoder' in modules:
            estimator = make_pipeline(encoder, estimator) 
        if 'trans' in modules:
            estimator = trans_target(estimator)
        if 'detector' in modules:
            estimator = super_detector(estimator)
        
        scores = cross_val_score(estimator, X_train, y_train, cv=cv, scoring=scoring)
        pipe_string = ' + '.join(modules + ('model',))
        print('{:.4f} ± {:.4f} [{}]'.format(np.mean(scores), np.std(scores), pipe_string))
best_estimator = trans_target(model)<jupyter_output><empty_output><jupyter_text>Submit (Out-of-Fold & Test Predictions)<jupyter_code>%%time
from sklearn.model_selection import cross_val_predict

y_oof = cross_val_predict(best_estimator, X_train, y_train, cv=cv)

y_oof = pd.Series(y_oof, index=X_train.index, name=target)
print(y_oof.head())
%%time
from sklearn.model_selection import cross_validate

scores = cross_validate(best_estimator, X_train, y_train, cv=cv, return_estimator=True)
y_subs = [estimator.predict(X_test) for estimator in scores['estimator']]
y_sub = np.mean(y_subs, axis=0)

y_sub = pd.Series(y_sub, index=X_test.index, name=target)
print(y_sub.head())
import os

path = os.path.join(os.getcwd(), 'pred')
if not os.path.exists(path):
    os.mkdir(path)

sub_path = os.path.join(path, '0 sub baseline.csv')
oof_path = os.path.join(path, '0 oof baseline.csv')

y_sub.to_csv(sub_path, header=True)
y_oof.to_csv(oof_path, header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13256484149855907, 'fraction_numerical': 0.029327004577046956, 'mean_word_length': 4.52434456928839, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '355029', 'license': 'MIT', 'n_tokens_mistral': 2316, 'n_tokens_neox': 2153, 'n_words': 422, 'path': 'examples/house/0 baseline.ipynb', 'repo_name': 'uberkinder/Robusta-AutoML', 'size': '5899'}