<jupyter_start><jupyter_text>Read input files<jupyter_code># Decided to use both input dataset 1 and 2 to get more data to train on
#df = pd.read_parquet("../container-data/input_dataset-1.parquet")
df = pd.read_parquet("../container-data/input_dataset-2.parquet")
pred_df = pd.read_parquet("../container-data/prediction_input.parquet")
pred_cols = ["Bolt_"+str(x)+"_Tensile" for x in range(1,7)]+list(pred_df.columns)
pred_cols
#df = pd.concat([df1.loc[:,pred_cols], df2.loc[:,pred_cols]], axis=0)
#df.to_pickle("../container-data/df.pkl")
#df = pd.read_pickle("../container-data/df.pkl")
df = df.loc[:,pred_cols]
df.columns<jupyter_output><empty_output><jupyter_text>Common preprocessing<jupyter_code># Want to extract the time of the earliest datapoint in the dataset
start_time = df.index.min()
start_time
df.head()
def common_prep(df):
    df["datetime"] = df.index.values
    df.index = np.arange(len(df))
    # We drop rows with missing. No use in trying to impute. Creates more uncertainty.
    df = df.dropna()
    return df
df = common_prep(df)<jupyter_output><empty_output><jupyter_text>Want to make a holdout dataset for validation. This is held separate from crossvalidation.The holdout-set is chosen as 20% of the data. (Not chunked in time.)Argue that we do not get data leakage from the future, but get better validation that we are able to model dependencies across the whole dataset.<jupyter_code>def split_trn_val(df, frac=0.2):
    df = df.copy()
    val_inds = np.random.choice(df.index.values, size=int(len(df)*frac), replace=False)
    val_df = df.iloc[df.index.isin(val_inds)]
    trn_df = df.iloc[~df.index.isin(val_inds)]
    return trn_df, val_df
trn_df, val_df = split_trn_val(df)
len(trn_df), len(val_df)
val_df_dt = val_df["datetime"]
#trn_df.to_pickle("../container-data/trn_df.pkl")
#val_df.to_pickle("../container-data/val_df.pkl")
y_cols = ["Bolt_"+str(count)+"_Tensile" for count in range(1,7)]
y_cols
def add_diff_cols(df):
    return df
def diff_bolt_pretension(df):
    pret = pd.read_csv("../container-data/bolt_pretension.csv", header=None)
    pret_dict = pd.Series(pret.iloc[:,1].values, index=pret.iloc[:,0]).to_dict()
    for y in y_cols:
        df[y] = df[y]-pret_dict[y.strip("_Tensile").replace("_", " ")]
    return df

def binarize_mode(df):
    df["mode"] = df["mode"].apply(lambda x: 1 if x=="start" else 0)
    return df

def categorize_mode(df):
    df["mode"] = df["mode"].apply(lambda x: "start" if x==1 else "operation")
    return df

def add_mode_switch(df):
    df["mode_switch"] = df["mode"]-(df["mode"].shift(fill_value=False))
    return df

def add_sec_since_switch(df):
    df["sec_since_switch"] = (df.groupby('mode')['datetime'].transform(lambda x: (x-x.min()).dt.seconds))
    return df

def add_days_since_start(df):
    df["days_since_start"] = (df["datetime"]-start_time).dt.days
    return df

def add_rolling_sum_speed(df, periods=[60,60*5, 60*60]):
    for period in periods:
        df["Turbine_Rotational Speed_sum"+str(period)] = df["Turbine_Rotational Speed"].rolling(period, min_periods=1).sum()
    return df

def remove_cols(df):
    remove_cols = ["datetime"]
    df = df.drop(remove_cols, axis=1)
    return df

def preprocessing(df):
    # Those that are commented out was left out as no improvement in cv score was shown
    #df = diff_bolt_pretension(df)
    df = binarize_mode(df)
    df = add_mode_switch(df)
    df = add_sec_since_switch(df)
    df = add_days_since_start(df)
    #df = add_rolling_sum_speed(df)
    df = remove_cols(df)
    df = categorize_mode(df)
    return df

def split_x_y(df):
    X = df.drop(y_cols, axis=1)
    y = df.loc[:, y_cols]
    return X, y
trn_df = preprocessing(trn_df)
df_X_train, df_y_train = split_x_y(trn_df)
val_df = preprocessing(val_df)
df_X_val, df_y_val = split_x_y(val_df)
x_cols = set([x for x in df_X_train.columns])-set(y_cols)
x_cols<jupyter_output><empty_output><jupyter_text>Make column selectors<jupyter_code>X = df_X_train
y = df_y_train
from sklearn.compose import make_column_selector
df.dtypes
cat_selector = make_column_selector(dtype_include=object)
num_selector = make_column_selector(dtype_include=np.number)
cat_selector(X), num_selector(X)<jupyter_output><empty_output><jupyter_text>Make column transformers  Tree preprocessor<jupyter_code>from sklearn.compose import make_column_transformer
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import OrdinalEncoder

cat_tree_processor = OrdinalEncoder(
    handle_unknown="use_encoded_value", unknown_value=-1
)
num_tree_processor = SimpleImputer(strategy="mean", add_indicator=True)

tree_preprocessor = make_column_transformer(
    (num_tree_processor, num_selector), (cat_tree_processor, cat_selector)
)
tree_preprocessor<jupyter_output><empty_output><jupyter_text>Linear preprocessor<jupyter_code>from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler

cat_linear_processor = OneHotEncoder(handle_unknown="ignore")
num_linear_processor = make_pipeline(
    StandardScaler(), SimpleImputer(strategy="mean", add_indicator=True)
)

linear_preprocessor = make_column_transformer(
    (num_linear_processor, num_selector), (cat_linear_processor, cat_selector)
)
linear_preprocessor<jupyter_output><empty_output><jupyter_text>Make pipelines  Lasso pipeline<jupyter_code>from sklearn.linear_model import LassoCV, RidgeCV, MultiTaskLassoCV

lasso_pipeline = make_pipeline(linear_preprocessor, LassoCV(n_jobs=4, cv=3, verbose=True, selection="random", max_iter=1000))
lasso_pipeline<jupyter_output><empty_output><jupyter_text>Random forest pipeline<jupyter_code>from sklearn.ensemble import RandomForestRegressor

rf_pipeline = make_pipeline(tree_preprocessor, RandomForestRegressor(n_estimators=3, n_jobs=6, random_state=42))
rf_pipeline<jupyter_output><empty_output><jupyter_text>Stacking regressor<jupyter_code>from sklearn.ensemble import StackingRegressor
from sklearn.linear_model import RidgeCV

estimators = [
    ("Lasso", lasso_pipeline),
    ("Random Forest", rf_pipeline),
]

stacking_regressor = StackingRegressor(estimators=estimators, final_estimator=RidgeCV(cv=KFold(n_splits=3, shuffle=False), scoring="neg_mean_absolute_percentage_error"))
stacking_regressor<jupyter_output><empty_output><jupyter_text>Cross-validation<jupyter_code>kf = KFold(n_splits=3, random_state=42, shuffle=True)
fold = 1
scores = {}
preds = {}
residuals = {}
trained_models = {}
for train_index, test_index in kf.split(df_X_train):
    scores[fold] = {}
    preds[fold] = {}
    residuals[fold] = {}
    trained_models[fold] = {}
    print(fold)
    X_train, X_test = df_X_train.iloc[train_index], df_X_train.iloc[test_index]
    for y in y_cols:
        print(y)
        scores[fold][y] = {}
        preds[fold][y] = {}
        residuals[fold][y] = {}
        trained_models[fold][y] = {}
        drop_cols = [x for x in y_cols if x!=y]
        print(drop_cols)
        y_train, y_test = df_y_train.iloc[train_index].drop(drop_cols, axis=1).values.flatten(), df_y_train.iloc[test_index].drop(drop_cols, axis=1)
        for (name, model) in estimators + [("Stacking Regressor", stacking_regressor)]:
            print(name)
            model.fit(X_train, y_train)
            y_pred = model.predict(X_test)
            score = mean_absolute_percentage_error(y_test, y_pred, multioutput="raw_values")
            print(score)
            scores[fold][y][name] = score
            preds[fold][y][name] = y_pred
            residuals[fold][y][name] = y_pred-y_test.values.flatten()
            trained_models[fold][y][name] = model
    fold = fold+1
residuals<jupyter_output><empty_output><jupyter_text>Make predictions on validation set<jupyter_code>scores = {}
preds = {}
residuals = {}
models = {}
for y in y_cols:
    print(y)
    drop_cols = [x for x in y_cols if x!=y]
    y_train = df_y_train.drop(drop_cols, axis=1).values.flatten()
    y_val = df_y_val.drop(drop_cols, axis=1).values.flatten()
    model = make_pipeline(linear_preprocessor, LassoCV(n_jobs=4, cv=3, verbose=True, selection="random", max_iter=1000))
    model.fit(df_X_train, y_train)
    y_pred = model.predict(df_X_val)
    score = mean_absolute_percentage_error(y_val, y_pred)
    print(score)
    scores[y] = score
    preds[y] = y_pred
    residuals[y] = y_pred-y_val
    models[y] = model<jupyter_output>Bolt_1_Tensile<jupyter_text>Investigate results<jupyter_code>for y in y_cols:
    a = models[y].steps[-1][1].alpha_
    coefs = models[y].steps[-1][1].coef_
    intercept = models[y].steps[-1][1].intercept_
    print(a)
    print(coefs)
    print(intercept)
scores
score_df = pd.DataFrame.from_dict(scores, orient="index")
score_df.plot(kind="bar", figsize=(24,18))
score_df.mean()<jupyter_output><empty_output><jupyter_text>Plot feature importances<jupyter_code>models
nrow= 3
ncol=2

# make a list of all dataframes 
fig, axes = plt.subplots(nrow, ncol, figsize=(30,24))

# plot counter
count=1
for r in range(nrow):
    for c in range(ncol):
        name = "Bolt_"+str(count)+"_Tensile"
        model = models[name]
        imps = pd.Series(model.feature_importances_, index=df_X_val.columns)
        imps.plot(kind="bar", ax=axes[r,c], title=name)
        count+=1<jupyter_output><empty_output><jupyter_text>It is interesting to see that days_since_start matters more for the bolts that are more difficult to predict.  Plot residuals Where do we miss?<jupyter_code>res_df = pd.DataFrame(data=np.transpose([residuals[y] for y in y_cols]), index=val_df_dt, columns=y_cols)
nrow= 3
ncol=2

# make a list of all dataframes 
fig, axes = plt.subplots(nrow, ncol, figsize=(30,24))

# plot counter
count=1
for r in range(nrow):
    for c in range(ncol):
        name = "Bolt_"+str(count)+"_Tensile"
        res_df[name].plot(style=".", ax=axes[r,c], title=name)
        count+=1<jupyter_output><empty_output><jupyter_text>Train final model with whole dataset<jupyter_code>df.columns
final_df = preprocessing(df)
df_X, df_y = split_x_y(final_df) 
df_X.columns, df_y.columns
fitted = {}
out_preds = {}
for y in y_cols:
    drop_cols = [x for x in y_cols if x!=y]
    y_train = df_y.drop(drop_cols, axis=1).values.flatten()
    model = make_pipeline(linear_preprocessor, LassoCV(n_jobs=4, cv=3, verbose=True, selection="random", max_iter=1000))
    model.fit(df_X, y_train)
    fitted[y] = model
    out_preds[y] = model.predict(df_X)
out_preds<jupyter_output><empty_output><jupyter_text>Apply same preprocessing to input prediction data<jupyter_code>pred_df = pd.read_parquet("../container-data/prediction_input.parquet")
pred_df_datetime = pred_df.index
pred_df = common_prep(pred_df)
pred_df = preprocessing(pred_df)<jupyter_output><empty_output><jupyter_text>Make predictions with final model for submission<jupyter_code>for y_col, pipeline in fitted.items():
    pred_df[y_col] = pipeline.predict(pred_df.loc[:,df_X.columns])
pred_df
pred_df = pred_df.set_index(pred_df_datetime)
pred_df
out_df = pred_df.loc[:,y_cols]
out_df
assert (pred_df_datetime == out_df.index).all()
pred_df.to_csv("../container-data/prediction_output_lasso.csv")
nrow= 3
ncol=2

# make a list of all dataframes 
fig, axes = plt.subplots(nrow, ncol, figsize=(30,24))

# plot counter
count=1
for r in range(nrow):
    for c in range(ncol):
        name = "Bolt_"+str(count)+"_Tensile"
        pred_df.loc[:,name].plot(style=".", ax=axes[r,c], title=name)
        count+=1
plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1277264002791834, 'fraction_numerical': 0.008811725702320711, 'mean_word_length': 5.2639344262295085, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '304170', 'license': 'MIT', 'n_tokens_mistral': 4309, 'n_tokens_neox': 4241, 'n_words': 791, 'path': 'notebooks/tht-cv.ipynb', 'repo_name': 'thomasht86/krafthack2022', 'size': '11462'}