<jupyter_start><jupyter_text>pytorch-tabnet* 5 fold cv* best result for single model alg.<jupyter_code>import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold

from pytorch_tabnet.tab_model import TabNetRegressor
# globals
FIGSIZE = (9, 6)
BATCH_SIZE = 256

SEED = 42

FILE_TRAIN = "train.csv"
# for adding features
def add_features(df):
    new_df = df.copy()
    new_df["datetime"] = pd.to_datetime(new_df["datetime"])

    # this way I add 3 engineered features
    new_df["hour"] = new_df["datetime"].dt.hour
    new_df["year"] = new_df["datetime"].dt.year

    return new_df
data_orig = pd.read_csv(FILE_TRAIN)

# feature engineering
data_added = add_features(data_orig)
all_columns = data_added.columns

# colonne da ignorare
# atemp and temp are strongly correlated (0.98) taking only one
del_columns = ["datetime", "casual", "registered", "temp"]

TARGET = "count"

cat_cols = ["season", "holiday", "workingday", "weather", "hour", "year"]

num_cols = list(set(all_columns) - set([TARGET]) - set(del_columns) - set(cat_cols))

features = sorted(cat_cols + num_cols)

print("Tutte le colonne:", len(all_columns))
print("Colonne ignorate:", len(del_columns))
print("target:", len([TARGET]))
print("Colonne cat:", len(cat_cols))
print("Colonne num:", len(num_cols))
print("Num. features", len(features))
data_used = data_added.drop(del_columns, axis=1)
# this part is required for TabNet
# categorical_columns = cat_cols
categorical_dims = {}
# save label encoder for predictions
vet_lenc = []

for col in cat_cols:
    # print(col, data_used[col].nunique(), data_used[col].unique())
    print(col)
    l_enc = LabelEncoder()
    data_used[col] = l_enc.fit_transform(data_used[col].values)
    vet_lenc.append(l_enc)
    categorical_dims[col] = len(l_enc.classes_)
cat_idxs = [i for i, f in enumerate(features) if f in cat_cols]
cat_dims = [categorical_dims[f] for i, f in enumerate(features) if f in cat_cols]<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>%%time

FOLDS = 7

skf = KFold(n_splits=FOLDS, shuffle=True, random_state=SEED)

# provato sembra meglio n_steps = 2
# forse perchè va in overfitting
params = {"n_steps": 2, "cat_dims": cat_dims, "cat_idxs": cat_idxs, "verbose": 50}

# we will save here all the results from FOLDS
best_models = []

EPOCHS = 1000
PATIENCE = 100

i = 1
for train_idx, valid_idx in skf.split(data_used):
    print()
    print("Processing fold:", i)

    data_train = data_used.iloc[train_idx]
    data_valid = data_used.iloc[valid_idx]

    x_train = data_train[features].values
    y_train = data_train[TARGET].values
    y_train = y_train.reshape(-1, 1)

    x_valid = data_valid[features].values
    y_valid = data_valid[TARGET].values
    y_valid = y_valid.reshape(-1, 1)

    model = TabNetRegressor(**params)

    # provo a cercare direttamente best su rmsle
    model.fit(
        x_train,
        y_train,
        eval_set=[(x_valid, y_valid)],
        eval_metric=["rmse", "rmsle"],
        max_epochs=EPOCHS,
        patience=PATIENCE,
        batch_size=BATCH_SIZE,
    )

    best_models.append(model)

    # next iteration
    i += 1<jupyter_output>Processing fold: 1
Device used : cuda
epoch 0  | loss: 64512.08286| val_0_rmse: 201.36648| val_0_rmsle: 3.71803 |  0:00:01s
epoch 50 | loss: 3015.46704| val_0_rmse: 51.37288| val_0_rmsle: 0.34843 |  0:00:50s
epoch 100| loss: 2562.25618| val_0_rmse: 50.92724| val_0_rmsle: 0.44655 |  0:01:38s
epoch 150| loss: 2425.86345| val_0_rmse: 51.12546| val_0_rmsle: 0.33278 |  0:02:26s
epoch 200| loss: 2400.93228| val_0_rmse: 48.65974| val_0_rmsle: 0.34055 |  0:03:14s

Early stopping occurred at epoch 201 with best_epoch = 101 and best_val_0_rmsle = 0.23017
Best weights from best epoch are automatically used!

Processing fold: 2
Device used : cuda
epoch 0  | loss: 64353.03053| val_0_rmse: 195.85575| val_0_rmsle: 1.98202 |  0:00:01s
epoch 50 | loss: 3121.24997| val_0_rmse: 52.79271| val_0_rmsle: 0.33041 |  0:00:51s
epoch 100| loss: 2678.63233| val_0_rmse: 51.7785 | val_0_rmsle: 0.23772 |  0:01:39s
epoch 150| loss: 2189.3605| val_0_rmse: 49.63096| val_0_rmsle: 0.23635 |  0:02:27s
epoch 200| loss: 22[...]<jupyter_text>Submission<jupyter_code>test_orig = pd.read_csv("test.csv")
# add engineered features
test_orig = add_features(test_orig)
# code categorical
for i, col in enumerate(cat_cols):
    print(col)
    l_enc = vet_lenc[i]
    test_orig[col] = l_enc.transform(test_orig[col].values)
x_test = test_orig[features].values

avg_score = np.zeros((x_test.shape[0], 1))

for i, model in enumerate(best_models):
    print()
    print("Predictions from model", i + 1)

    score_test = model.predict(x_test)

    avg_score += score_test / float(FOLDS)
df_sub = pd.read_csv("sampleSubmission.csv")
df_sub["count"] = avg_score
# replace negative with zero
condition = df_sub["count"] < 0

df_sub.loc[condition, "count"] = 0
FILE_SUB = "submission34.csv"

df_sub.to_csv(FILE_SUB, index=False)
!kaggle competitions submit -c "bike-sharing-demand" -f $FILE_SUB -m "sub34 tabnet cv 7-folds"<jupyter_output>100%|█████████████████████████████████████████| 244k/244k [00:01<00:00, 146kB/s]
Successfully submitted to Bike Sharing Demand
{'fraction_non_alphanumeric': 0.13172506222477504, 'fraction_numerical': 0.06988320888378327, 'mean_word_length': 4.78516057585825, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '438686', 'license': 'MIT', 'n_tokens_mistral': 2297, 'n_tokens_neox': 2023, 'n_words': 463, 'path': 'tabnet2-cv.ipynb', 'repo_name': 'luigisaetta/bike-sharing-forecast', 'size': '5223'}