<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;References2&nbsp;&nbsp;Load Data3&nbsp;&nbsp;Preprocessing4&nbsp;&nbsp;Folds5&nbsp;&nbsp;Training5.1&nbsp;&nbsp;Preparation5.2&nbsp;&nbsp;Deep multilabel model keras5.3&nbsp;&nbsp;Deep multilabel model torch5.4&nbsp;&nbsp;Term model5.5&nbsp;&nbsp;Zero class prediction model5.6&nbsp;&nbsp;Error class prediction model5.7&nbsp;&nbsp;Calibration5.8&nbsp;&nbsp;Blender model5.9&nbsp;&nbsp;Manual tuning6&nbsp;&nbsp;Public models6.1&nbsp;&nbsp;keras NN +PCA with Label smoothing CV[0.01562] LB [0.01859]6.2&nbsp;&nbsp;Pytorch-RankGauss-PCA-NN CV [0.014572] LB [0.01839]6.3&nbsp;&nbsp;MODEL1 CV [0.01562060391771847] LB [0.01833]7&nbsp;&nbsp;Auto Tuning7.1&nbsp;&nbsp;Error class7.2&nbsp;&nbsp;Zero class7.3&nbsp;&nbsp;Blender7.4&nbsp;&nbsp;Run!8&nbsp;&nbsp;Final prediction9&nbsp;&nbsp;Submission10&nbsp;&nbsp;Error analysis11&nbsp;&nbsp;Offline vs Public  Referenceshttps://www.kaggle.com/sinamhd9/mechanisms-of-action-moa-tutorial/datascalers: https://www.kaggle.com/liuhdme/moa-competitionkeras: https://www.kaggle.com/riadalmadani/keras-nn-pca-with-label-smoothing/commentsblending: https://www.kaggle.com/c/lish-moa/notebooks?competitionId=19988&sortBy=scoreDescendingClasses info:https://docs.google.com/spreadsheets/d/1NVPfqcJKWd-Oes610N-wHMYOKpO2WiH6PU7wFqZyNX8 Что можно поделать:1. (+) Модель предсказывающая все нули или нет2. Модель определения termов из названий классов3. Кластеризация + Y-decode4. G,C - autoencoders5. Imbalance6. Class cleaning, только в train!7. (+) Отдельная модель на классы с ошибками  Load Data<jupyter_code># Importing useful libraries
# pip install jupyter_contrib_nbextensions lightgbm iterative-stratification tensorflow tensorflow-addons mlflow kaggle jupyter hyperopt

import pickle
from os.path import isfile
from functools import partial
import warnings
warnings.filterwarnings("ignore")

# Adding iterative-stratification 
# Select add data from the right menu and search for iterative-stratification, then add it to your kernel.
import sys
import os
import random
sys.path.append('../input/iterative-stratification/iterative-stratification-master')

from iterstrat.ml_stratifiers import MultilabelStratifiedKFold

sys.path.append('../input/rank-gauss')
from gauss_rank_scaler import GaussRankScaler

from time import time
import datetime
import gc
import collections

import numpy as np
import pandas as pd 

# ML tools 
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras import layers
import tensorflow.keras.backend as K
from sklearn.metrics import log_loss
from tensorflow_addons.layers import WeightNormalization

# Setting random seeds
def seed_everything(seed=42):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    tf.random.set_seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True

seed_everything(seed=42)

# Visualization tools
import seaborn as sns
import matplotlib.pyplot as plt
sns.set_style('white')
sns.set(font_scale=1.2)

import requests
has_internet = True
try:
    r = requests.get('http://datadigger.ru', timeout=3)
    !pip install -q mlflow
    import mlflow
    
    mlflow.set_tracking_uri('http://datadigger.ru:5000')

except Exception:
    has_internet = False

def dict_flatten(d, parent_key='', sep='_'):
    items = []
    for k, v in d.items():
        new_key = parent_key + sep + k if parent_key else k
        if isinstance(v, collections.MutableMapping):
            items.extend(dict_flatten(v, new_key, sep=sep).items())
        else:
            items.append((new_key, v))
    return dict(items)

df_train = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
print('train data size', df_train.shape)

df_drug = pd.read_csv('/kaggle/input/lish-moa/train_drug.csv')
print('train drug size', df_drug.shape)

df_target_ns = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
print('train target nonscored size', df_target_ns.shape)

df_target_s = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
print('train target scored size', df_target_s.shape)

df_target_s = df_target_s.merge(df_drug, on=['sig_id'])
print('train target scored size', df_target_s.shape)

df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
print('test data size', df_test.shape)

df_sample = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
print('sample submission size', df_sample.shape)

def g_c_features(df):
    g_features = [cols for cols in df.columns if cols.startswith('g-')]
    c_features = [cols for cols in df.columns if cols.startswith('c-')]
    return g_features, c_features

def preprocess(df):
    df['cp_time'] = df['cp_time'].map({24:1, 48:2, 72:3})
    df['cp_dose'] = df['cp_dose'].map({'D1':0, 'D2':1})
    df['cp_type'] = df['cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
    return df

X = preprocess(df_train)
X_test = preprocess(df_test)

ind_te = X_test[X_test['cp_type']==1].index

y = df_target_s.drop('sig_id', axis=1)
y0 =  df_target_ns.drop('sig_id', axis=1)

print('New data shape', X.shape)

def perc_empty(df):
    df = df.drop(['drug_id'], axis=1, errors='ignore')
    return 100 * (1 - len(df[(df.T != 0).any()]) / len(df))

print("Scored без класса: {}%".format(round(perc_empty(y), 3)))
print("NoScored без класса: {}%".format(round(perc_empty(y0), 3)))
print("Нет никакого класса: {}%".format(round(perc_empty(pd.concat([y, y0], axis=1)), 3)))<jupyter_output>train data size (23814, 876)
train drug size (23814, 2)
train target nonscored size (23814, 403)
train target scored size (23814, 207)
train target scored size (23814, 208)
test data size (3982, 876)
sample submission size (3982, 207)
New data shape (21948, 875)
Scored без класса: 34.176%
NoScored без класса: 79.087%
Нет никакого класса: 16.694%<jupyter_text>Preprocessing<jupyter_code>from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer, StandardScaler
from sklearn.cluster import KMeans
from sklearn.feature_selection import VarianceThreshold

def variance_reduction(X, X_test, **params):
    if not params.get('enabled', True):
        return X, X_test
    thresh = params.get('threshold', 0.8)

    columns_to_skip = [
        c for c in ['drug_id', 'cp_type', 'cp_time','cp_dose'] 
        if c in set(X.columns)
    ]
    cols_num = len(columns_to_skip)
    
    var_thresh = VarianceThreshold(thresh)
    data = X.append(X_test)
    try:
        data_transformed = var_thresh.fit_transform(data.iloc[:, cols_num:])
    except Exception as e:
        print(e, str(thresh))
        return X, X_test

    train_features_transformed = data_transformed[ : X.shape[0]]
    test_features_transformed = data_transformed[-X_test.shape[0] : ]

    X = pd.DataFrame(
        X[columns_to_skip].values.reshape(-1, cols_num),
        columns=columns_to_skip
    )

    X = pd.concat([X, pd.DataFrame(train_features_transformed)], axis=1)

    X_test = pd.DataFrame(
        X_test[columns_to_skip].values.reshape(-1, cols_num),
        columns=columns_to_skip
    )

    X_test = pd.concat([X_test, pd.DataFrame(test_features_transformed)], axis=1)
    return X, X_test

def fe_cluster(train, test, **params):
    if not params.get('enabled', True):
        return train, test
    n_clusters_g = params.get('n_clusters_g', 35) 
    n_clusters_c = params.get('n_clusters_c', 5)
    random_state = params.get('seed', 299)
    
    features_g, features_c = g_c_features(train)
    def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
        train_ = train[features].copy()
        test_ = test[features].copy()
        data = pd.concat([train_, test_], axis = 0)
        kmeans = KMeans(n_clusters = n_clusters, random_state = random_state).fit(data)
        train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
        test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
        train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
        test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
        return train, test
    
    train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
    train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
    return train, test

def quantile_transformer(df, df_test, **params):
    random_state = params.get('seed', 42)
    g_features, c_features = g_c_features(df)

    for col in (g_features + c_features):
        transformer = QuantileTransformer(n_quantiles=100, random_state=random_state, output_distribution='normal')
        vec_len = len(df[col].values)
        raw_vec = df[col].values.reshape(vec_len, 1)
        vec_len_test = len(df_test[col].values)
        raw_vec_test = df_test[col].values.reshape(vec_len_test, 1)
        transformer.fit(raw_vec)
        df[col] = transformer.transform(raw_vec).reshape(1, vec_len)[0]
        df_test[col] = transformer.transform(raw_vec_test).reshape(1, vec_len_test)[0]
    return df, df_test

def rank_gauss(df, **params):
    g_features, c_features = g_c_features(df)
    cols_numeric = g_features + c_features
    df[cols_numeric] = GaussRankScaler().fit_transform(df[cols_numeric])
    return df

def standard_scaler(df, df_test, **params):
    g_features, c_features = g_c_features(df)
    cols_numeric = g_features + c_features
    scaler = StandardScaler()
    df[cols_numeric] = scaler.fit_transform(df[cols_numeric])
    df_test[cols_numeric] = scaler.transform(df_test[cols_numeric])
    return df, df_test

def fe_cluster_pca(train, test, n_clusters=5, seed = 42):
    pca_g_cols = [c for c in train.columns if c.startswith('pca_g-')]
    pca_c_cols = [c for c in train.columns if c.startswith('pca_c-')]
    train_gpca = train[pca_g_cols]
    test_gpca = test[pca_g_cols]
    train_cpca = train[pca_c_cols]
    test_cpca = test[pca_c_cols]

    train_pca=pd.concat((train_gpca,train_cpca),axis=1)
    test_pca=pd.concat((test_gpca,test_cpca),axis=1)

    data=pd.concat([train_pca,test_pca],axis=0)
    kmeans = KMeans(n_clusters = n_clusters, random_state = seed).fit(data)

    train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
    test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]

    train = pd.get_dummies(train, columns = [f'clusters_pca'])
    test = pd.get_dummies(test, columns = [f'clusters_pca'])
    return train, test

def pca_transformer(X, X_test, **params):
    # Please see reference 3 for this part
    if not params.get('enabled', True):
        return X, X_test
    random_state = params.get('seed', 42)
    n_comp_cells = params.get('n_comp_cells', 0.95)
    n_comp_genes = params.get('n_comp_genes', 0.95)
#     print(f'pca {n_comp_cells}, {n_comp_genes}') 
    g_features, c_features = g_c_features(X)

    data = pd.concat([pd.DataFrame(X[g_features]), pd.DataFrame(X_test[g_features])])
    data2 = (PCA(n_comp_genes, random_state=random_state).fit_transform(data[g_features]))
    train2 = data2[:X.shape[0]]
    test2 = data2[-X_test.shape[0]:]

    train2 = pd.DataFrame(train2, columns=[f'pca_g-{i}' for i in range(data2.shape[1])])
    test2 = pd.DataFrame(test2, columns=[f'pca_g-{i}' for i in range(data2.shape[1])])

    X = pd.concat((X, train2), axis=1)
    X_test = pd.concat((X_test, test2), axis=1)

    data = pd.concat([pd.DataFrame(X[c_features]), pd.DataFrame(X_test[c_features])])
    data2 = (PCA(n_comp_cells, random_state=random_state).fit_transform(data[c_features]))
    train2 = data2[:X.shape[0]]
    test2 = data2[-X_test.shape[0]:]

    train2 = pd.DataFrame(train2, columns=[f'pca_c-{i}' for i in range(data2.shape[1])])
    test2 = pd.DataFrame(test2, columns=[f'pca_c-{i}' for i in range(data2.shape[1])])

    X = pd.concat((X, train2), axis=1)
    X_test = pd.concat((X_test, test2), axis=1)

    clusters = params.get('n_clusters', 0)
    if clusters:
        X, X_test = fe_cluster_pca(X, X_test, n_clusters=clusters, seed=random_state)
    return X, X_test

gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']

def fe_stats(train, test, **params):

    if not params.get('enabled', True):
        return train, test

    features_g, features_c = g_c_features(train)
    for df in train, test:
        df['g_sum'] = df[features_g].sum(axis = 1)
        df['g_mean'] = df[features_g].mean(axis = 1)
        df['g_std'] = df[features_g].std(axis = 1)
        df['g_kurt'] = df[features_g].kurtosis(axis = 1)
        df['g_skew'] = df[features_g].skew(axis = 1)
        df['c_sum'] = df[features_c].sum(axis = 1)
        df['c_mean'] = df[features_c].mean(axis = 1)
        df['c_std'] = df[features_c].std(axis = 1)
        df['c_kurt'] = df[features_c].kurtosis(axis = 1)
        df['c_skew'] = df[features_c].skew(axis = 1)
        df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
        df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
        df['gc_std'] = df[features_g + features_c].std(axis = 1)
        df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
        df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
    
        df['c52_c42'] = df['c-52'] * df['c-42']
        df['c13_c73'] = df['c-13'] * df['c-73']
        df['c26_c13'] = df['c-23'] * df['c-13']
        df['c33_c6'] = df['c-33'] * df['c-6']
        df['c11_c55'] = df['c-11'] * df['c-55']
        df['c38_c63'] = df['c-38'] * df['c-63']
        df['c38_c94'] = df['c-38'] * df['c-94']
        df['c13_c94'] = df['c-13'] * df['c-94']
        df['c4_c52'] = df['c-4'] * df['c-52']
        df['c4_c42'] = df['c-4'] * df['c-42']
        df['c13_c38'] = df['c-13'] * df['c-38']
        df['c55_c2'] = df['c-55'] * df['c-2']
        df['c55_c4'] = df['c-55'] * df['c-4']
        df['c4_c13'] = df['c-4'] * df['c-13']
        df['c82_c42'] = df['c-82'] * df['c-42']
        df['c66_c42'] = df['c-66'] * df['c-42']
        df['c6_c38'] = df['c-6'] * df['c-38']
        df['c2_c13'] = df['c-2'] * df['c-13']
        df['c62_c42'] = df['c-62'] * df['c-42']
        df['c90_c55'] = df['c-90'] * df['c-55']
        for feature in features_c:
             df[f'{feature}_squared'] = df[feature] ** 2     

        for feature in gsquarecols:
            df[f'{feature}_squared'] = df[feature] ** 2
    return train, test

def preprocess_X(params, X, X_test, y, y0, seed=42):
    p = params
    p_scaler = p['scaler']
    p_pca = p['pca']
    p_fe_cluster = p['fe_cluster']
    p_fe_stats = p['fe_stats']
    p_variance_reduction = p['variance_reduction']
#     print(X.shape, 'initial')
    if p_scaler == 'quantile':
        X, X_test = quantile_transformer(X, X_test, seed=seed)
    elif p_scaler == 'gauss':
        X = rank_gauss(X)
        X_test = rank_gauss(X_test)
    elif p_scaler == 'standard':
        X, X_test = standard_scaler(X, X_test)
    elif p_scaler != 'none':
        raise Exception(f'Unknown scaler: {p_scaler}')
#     print(X.shape, 'scaler')
    X, X_test = pca_transformer(X, X_test, seed=seed, **p_pca)
#     print(X.shape, 'pca')
    X, X_test = fe_cluster(X, X_test, seed=seed, **p_fe_cluster)
#     print(X.shape, 'cluster')
    X, X_test = fe_stats(X, X_test, **p_fe_stats)
#     print(X.shape, 'fe_stats')
    X, X_test = variance_reduction(X, X_test, **p_variance_reduction)
#     print(X.shape, 'variance')
    if p.get("shuffle_cols", True):
        X, X_test = shuffle_cols(X, X_test)

    y0 = y0[X['cp_type'] == 0].reset_index(drop = True)
    y = y[X['cp_type'] == 0].reset_index(drop = True)
    X = X[X['cp_type'] == 0].reset_index(drop = True)

    X = X.drop(['sig_id'], axis=1)
    X_test = X_test.drop(['sig_id'], axis=1)
    return X, X_test, y, y0

def shuffle_cols(train, test):
    # В зависимости от seed перемешиваем фичи
    features_shrink = 1.
    inp_size = int(np.ceil(features_shrink * len(train.columns)))
    split_cols = np.random.choice(train.columns, inp_size, replace=False)
    return train[split_cols], test[split_cols]<jupyter_output><empty_output><jupyter_text>Folds<jupyter_code>from sklearn.model_selection import KFold

def train_test_split(X_f, y_f, n_split=7, seed=42):
    return fold(X_f, y_f, n_split, seed)[0]

def fold_simple(X_f, y_f, n_split, seed):
    if len(y_f.columns) > 1:
        f = MultilabelStratifiedKFold(n_splits = n_split, random_state = seed, shuffle = True)
        return list(f.split(X_f, y_f))
    else:
        f = StratifiedKFold(n_splits = n_split, random_state=seed, shuffle = True)
        return list(f.split(X_f, y_f))

def fold_drug(X_f, y_f, drug_thresh=18, n_split=7, seed=42):
    vc = y_f.drug_id.value_counts()
    vc1 = vc.loc[vc <= drug_thresh].index.sort_values()
    vc2 = vc.loc[vc > drug_thresh].index.sort_values()
    
    target_cols = [c for c in y_f.columns if c != 'drug_id']

    # Сначала бьём на фолды лекарства
    # STRATIFY DRUGS 18X OR LESS
    skf1 = MultilabelStratifiedKFold(n_splits=n_split, shuffle=True, random_state=seed)
    tmp1 = y_f.groupby('drug_id').mean().loc[vc1]
    split_1 = list(skf1.split(tmp1, tmp1[target_cols]))

    # STRATIFY DRUGS MORE THAN 18X
    skf2 = MultilabelStratifiedKFold(n_splits=n_split, shuffle=True, random_state=seed)
    tmp2 = y_f.loc[y_f.drug_id.isin(vc2)].reset_index()
    split_2 = list(skf2.split(tmp2[target_cols], tmp2[target_cols]))

    folds = []
    for i in range(n_split):
        ind_tr_drug, ind_val_drug = split_1[i]
        tr_drug, val_drug = tmp1.iloc[ind_tr_drug].index, tmp1.iloc[ind_val_drug].index
        ind_tr_1, ind_val_1 = y_f.loc[y_f.drug_id.isin(tr_drug)].index, y_f.loc[y_f.drug_id.isin(val_drug)].index
        
        ind_tr_2, ind_val_2 = split_2[i]
        ind_tr_2, ind_val_2 = tmp2.iloc[ind_tr_2]['index'], tmp2.iloc[ind_val_2]['index']

        ind_tr = np.concatenate([ind_tr_1, ind_tr_2])
        ind_val = np.concatenate([ind_val_1, ind_val_2])
        folds.append((ind_tr, ind_val))
    return folds

fold = fold_simple<jupyter_output><empty_output><jupyter_text>Training  Preparation<jupyter_code>from tensorflow.keras import regularizers
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
from sklearn.model_selection import StratifiedKFold

p_min = 1e-10
p_max = 1-1e-10

def y_to_zero_class(df_y):
    return pd.DataFrame(df_y.max(axis=1).map({1: 0, 0: 1}))

def log_loss_metric(y_true, y_pred, columns=None):
    metrics = []
    y_pred = np.clip(y_pred, p_min, p_max)
    cols = y_true.columns if columns is None else columns
    for _target in cols:
        metrics.append(
            log_loss(
                y_true.loc[:, _target],
                y_pred.loc[:, _target].astype(float),
                labels = [0, 1]
            )
        )
    return np.mean(metrics)

def log_loss_result(y_true, y_pred):
    return log_loss_metric(y_true, y_pred, columns=y.columns)

def logloss(y_true, y_pred):
    y_pred = tf.clip_by_value(y_pred,p_min,p_max)
    return -K.mean(y_true*K.log(y_pred) + (1-y_true)*K.log(1-y_pred))

checkpoint_path = "model.h5"

def callbacks(verbose=0):
    rlr = ReduceLROnPlateau(
        monitor = 'val_logloss', factor = 0.1, patience = 3, verbose = verbose, 
        min_delta=1e-4, mode = 'min'
    )
#     ckp = ModelCheckpoint(
#         checkpoint_path, monitor = 'val_logloss', verbose = 0, 
#         save_best_only = True, mode = 'min'
#     )
    es = EarlyStopping(
        monitor = 'val_logloss', min_delta = 1e-5, patience = 10, mode = 'min', 
        baseline = None, restore_best_weights = True, verbose = verbose
    )
    return rlr, es#, ckp

def y_arr_to_df(y_arr, cols=None):
    if cols is None:
        cols = y.columns
    return pd.DataFrame(y_arr, columns=cols)

error_classes = [
    'cyclooxygenase_inhibitor',
    'dopamine_receptor_antagonist',
    'glutamate_receptor_antagonist',
    'adrenergic_receptor_antagonist',
    'dna_inhibitor'
]
def y_to_error_classes(y_df):
#     y_df_wo = pd.DataFrame(y_df.drop(error_classes, axis=1).max(axis=1))
#     y_df_wo.columns = ['other']
    y_df_w = y_df[error_classes]
#     y_res = pd.concat([y_df_wo, y_df_w], axis=1)
    return y_df_w

y_to_error_classes(y).sum(axis=0)<jupyter_output><empty_output><jupyter_text>Deep multilabel model keras<jupyter_code>from sklearn.utils import class_weight

batch_size = 128

def create_model(
    num_cols_x, num_cols_y, hid_layers, 
    activations, dropout_rate, 
    lr, label_smoothing, 
    weight_decay=1e-5,
    batch_norm=True, weight_norm=True
):
    inp1 = tf.keras.layers.Input(shape = (num_cols_x, ))
    x1 = inp1
    if batch_norm:
        x1 = tf.keras.layers.BatchNormalization()(x1)
    x1 = tf.keras.layers.Dropout(dropout_rate[0])(x1)

    for i, units in enumerate(hid_layers):
        activation = activations[i]

        if activation == 'leaky_relu':
            dense = tf.keras.layers.Dense(units)
        else:
            dense = tf.keras.layers.Dense(units, activation=activation)

        if weight_norm and weight_norm != 'output':
            x1 = WeightNormalization(dense)(x1)
        else:
            x1 = dense(x1)

        if activation == 'leaky_relu':
            x1 = tf.keras.layers.LeakyReLU(alpha=0.01)(x1)
        
        x1 = tf.keras.layers.Dropout(dropout_rate[i])(x1)
        if batch_norm:
            x1 = tf.keras.layers.BatchNormalization()(x1)
    
    
    out_dense = tf.keras.layers.Dense(num_cols_y, activation='sigmoid')
    if weight_norm:
        x1 = WeightNormalization(out_dense)(x1)
    else:
        x1 = out_dense(x1)
    
    model = tf.keras.models.Model(inputs=inp1, outputs=x1)    
    opt = tfa.optimizers.AdamW(learning_rate=lr, weight_decay=weight_decay)
#     opt = tfa.optimizers.Lookahead(opt, sync_period=10)
    model.compile(
        optimizer=opt,
        loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing), 
        metrics=logloss
    )
    
    return model

EPOCHS = None

class DeepMultiLabelModel(object):
    def __init__(self, params, model_name, weights_from=None, verbose=0, seed=42):
        self.deep_params = params
        self.model_name = model_name
        self.seed = seed
        self.num_epochs = EPOCHS or params['epochs']
        self.model = None
        self.history = None
        self.weights_from = None
        self.verbose = verbose
        self.cv_models = []
        self.classes = []
    
    @property
    def real_model(self):
        if self.model is not None:
            return self
        if self.cv_models:
            return self.cv_models[0]
        return None

    @property
    def definition(self):
        model = self.real_model
        model_info = []
        if model:
            model.model.summary(print_fn=model_info.append)
        return "\n".join(model_info)
    
    @property
    def train_history(self):
        model = self.real_model
        if model is None:
            return None
        return model.history
    
    def fit(self, x_tr, y_tr, x_val, y_val, y0_tr, y0_val):
        inp_size = x_tr.shape[1]
        y_shape = y_val.shape[1]
        hid_layer = self.deep_params['hid_layer']
        activation = self.deep_params['activation']
        dropout = self.deep_params['dropout']
        learning_rate = self.deep_params['learning_rate']
        label_smoothing = self.deep_params['label_smoothing']
        batch_norm = self.deep_params.get('batch_norm', True)
        weight_norm = self.deep_params.get('weight_norm', True)
        init_non_scored_weights = self.deep_params.get('init_non_scored_weights', True)
        cls_weight = self.deep_params.get('class_weight')
        weight_decay = self.deep_params.get('weight_decay', 1e-5)

        # Scored модель
        model = create_model(
            inp_size, y_shape, hid_layer, activation, dropout, 
            learning_rate, label_smoothing,
            weight_decay=weight_decay,
            batch_norm=batch_norm, weight_norm=weight_norm
        )
        
        if init_non_scored_weights:
            ns_y_tr = y0_tr
            ns_y_val = y0_val
            if init_non_scored_weights == 'ALL_TARGETS':
                ns_y_tr = np.hstack([y_tr, y0_tr])
                ns_y_val = np.hstack([y_val, y0_val])

            # Non-scored модель
            model0 = create_model(
                inp_size, ns_y_val.shape[1], hid_layer, activation, dropout, 
                learning_rate, label_smoothing,
                weight_decay=weight_decay,
                batch_norm=batch_norm, weight_norm=weight_norm
            )
            model0.fit(
                x_tr, ns_y_tr, validation_data=(x_val, ns_y_val),
                epochs = self.num_epochs, batch_size = batch_size,
                callbacks = callbacks(self.verbose), verbose = self.verbose
            )
            # Transfer weights
            for i in range(len(model.layers)-1):
                model.layers[i].set_weights(model0.layers[i].get_weights())

        self.history = model.fit(
            x_tr, y_tr, validation_data=(x_val, y_val),
            epochs = self.num_epochs, batch_size = batch_size,
            callbacks = callbacks(self.verbose), verbose = self.verbose,
            class_weight = cls_weight
        )
        self.model = model

    def predict(self, X):
        if isinstance(X, pd.DataFrame):
            X = X.astype('float64').values
        if self.cv_models:
            preds = []
            for model in self.cv_models:
                preds.append(model.predict(X))
            return np.mean(preds, axis=0)
        else:
            return self.model.predict(X)
        
    def cv(self, X, y, y0, run_name=None, n_split=7, return_pred=False,
           metric_fn=log_loss_metric, overfit=True, run_tags=None, max_score=None):
        seed = self.seed
        self.cv_models = []
        splits = fold(X, y, n_split=n_split, seed=self.seed)
        
        y = y.drop(columns=['drug_id'], errors='ignore')
        y0 = y0.drop(columns=['drug_id'], errors='ignore')

        ycols = y.columns
        self.classes = ycols
        yvals = y.astype(float).values
        y0vals = y0.astype(float).values
        X_vals = X.astype('float64').values

        test_pred = pd.DataFrame(index=X.index, columns=ycols)
        test_pred.loc[:,:] = 0
        model_def = None
        
        initial_time = time()
        for n, (tr, te) in enumerate(splits):
            start_time = time()
            # Обучающая,Валидационная выборка
            x_tr, x_test = X_vals[tr], X_vals[te]
            # Y предварительной задачи (non-scored)
            y0_tr, y0_test = y0vals[tr], y0vals[te]
            # Y основной задачи
            y_tr, y_test = yvals[tr], yvals[te]

            # Разбиваем обучение еще на train/val
            if not overfit:
                ind_tr, ind_val = train_test_split(pd.DataFrame(x_tr), pd.DataFrame(y_tr))
                x_tr, x_val = x_tr[ind_tr], x_tr[ind_val]
                y_tr, y_val = y_tr[ind_tr], y_tr[ind_val]
                y0_tr, y0_val = y0_tr[ind_tr], y0_tr[ind_val]
            else:
                x_val = x_test
                y_val = y_test
                y0_val = y0_test

            model = DeepMultiLabelModel(
                self.deep_params, model_name=self.model_name, 
                verbose=self.verbose, seed=self.seed
            )
            model.fit(x_tr, y_tr, x_val, y_val, y0_tr, y0_val)
            model_def = model.definition

            test_pred.loc[te, ycols] = model.predict(x_test)

            self.cv_models.append(model)

            oof = metric_fn(y.loc[te, ycols], test_pred.loc[te, ycols])
            print(f'[{str(datetime.timedelta(seconds = time() - start_time))[2:7]}], Fold {n}: {oof}')
            if max_score and oof > max_score:
                print(f'break cv execution {oof} > {max_score}')
                if not return_pred:
                    return None
                else:
                    return None, None

        logloss_valid = metric_fn(y, test_pred)
        print(f'Valid logloss: {logloss_valid}')
        if has_internet and run_name:
            mlflow.set_experiment('Kaggle-MOA-{}'.format(self.model_name))
            with mlflow.start_run(run_name=run_name):
                mlflow.log_params(dict_flatten({
                    'n_split': n_split,
                    'p_min': p_min,
                    'p_max': p_max,
                    'nn': self.deep_params
                }))
                mlflow.log_metric(key="logloss_valid", value=logloss_valid)
                run_tags = run_tags or {}
                run_tags.update({
                    'model_def': model_def,
                    'run': run_name,
                    'run_time': time() - initial_time
                })
                mlflow.set_tags(run_tags)
        if not return_pred:
            return logloss_valid
        else:
            return logloss_valid, test_pred
    
    def errors(self, X, y):
        # Количество ошибок, когда должно быть везде 0, а на самом деле - нет
        # Количество ошибок, когда должно что-то быть, а на самом деле везде 0
        # Для уменьшения этих ошибок можно применять lgb_zero
        y = y.drop(columns=['drug_id'], errors='ignore')
        ycols = y.columns
        seed_everything(self.seed)

        _, te = train_test_split(X, y, n_split=5)

        x_val = X_p.astype('float64').values[te]

        y_true = y.loc[te, ycols].reset_index(drop=True)
        y_pred = y_arr_to_df(self.predict(x_val), ycols)

        non_zeros = y_true[(y_true.T != 0).any()].index
        all_zeros = y_true[~(y_true.T != 0).any()].index

        clip_p_min = 1e-5
        clip_p_max = 1 - 1e-5

        y_pred_clip = np.clip(y_pred, clip_p_min, clip_p_max)

        print('Logloss:', log_loss_metric(y_true, y_pred))
        print('Logloss all zeros:', log_loss_metric(y_true.loc[all_zeros, :], y_pred.loc[all_zeros, :]))
        print('Logloss non zeros:', log_loss_metric(y_true.loc[non_zeros, :], y_pred.loc[non_zeros, :]))
        print('Logloss clip:', log_loss_metric(y_true, y_pred_clip))
        print('Logloss all zeros clip:', log_loss_metric(y_true.loc[all_zeros, :], y_pred_clip.loc[all_zeros, :]))
        print('Logloss non zeros clip:', log_loss_metric(y_true.loc[non_zeros, :], y_pred_clip.loc[non_zeros, :]))

        losses = []
        for i in range(y_true.shape[1]):
            y_true_cl = y_true.iloc[:,i]
            y_pred_cl = y_pred.iloc[:,i]
            losses.append({
                "index": i,
                "class": y_true_cl.name,
                'true_0': len(y_true_cl[y_true_cl == 0]),
                'true_1': len(y_true_cl[y_true_cl == 1]),
                "loss": log_loss(y_true_cl.values, y_pred_cl.values, labels=[0, 1]),
                'pred_hist_0': y_pred_cl[y_pred_cl <= 0.5].round(1).value_counts().sort_index().reset_index().values,
                'pred_hist_1': y_pred_cl[y_pred_cl > 0.5].round(1).value_counts().sort_index().reset_index().values,
            })

        return pd.DataFrame(losses).set_index(['index', 'class']).sort_values('loss', ascending=False)<jupyter_output><empty_output><jupyter_text>Deep multilabel model torch  Term model<jupyter_code>def split_to_terms(s):
    return s.split('_')

terms = pd.DataFrame({'terms': y.columns.map(split_to_terms)}).explode('terms')['terms'].value_counts()
terms = list(terms[terms > 1].index)
terms_map = {t: i for i, t in enumerate(terms)}

def y_to_terms(y_df):
    term_vals = []
    for _, row in y_df.iterrows():
        new_classes = [0] * len(terms)
        terms_1 = set(pd.DataFrame(row[row > 0].index.map(split_to_terms)).explode(0)[0].values.tolist())
        for term in terms_1:
            if term not in terms_map:
                continue
            new_classes[terms_map[term]] = 1
        term_vals.append(new_classes)
    return pd.DataFrame(term_vals, columns=terms)
# preprocess_params, deep_params, _, _, _ = ({'fe_cluster': {'enabled': True, 'n_clusters_c': 6, 'n_clusters_g': 44}, 'fe_stats': {'enabled': False}, 'pca': {'enabled': False}, 'scaler': 'none', 'shuffle_cols': True, 'variance_reduction': {'enabled': True, 'threshold': 0.9585049261745544}, 'use_zero_pred_model': True}, {'activation': ('selu', 'swish', 'swish'), 'dropout': (0.7, 0.7, 0.3), 'hid_layer': (1152, 1152, 2048), 'init_non_scored_weights': False, 'label_smoothing': 0.0007000000000000001, 'learning_rate': 0.016, 'epochs': 500}, {'threshold': 0}, 29, 1)

# term_model = DeepMultiLabelModel(deep_params, 'terms')

# y_term = y_to_terms(pd.concat([y, y0], axis=1))
# term_model.cv(X_p, y_term, y0, n_split=5)<jupyter_output><empty_output><jupyter_text>Zero class prediction model<jupyter_code># X_p, X_test_p = preprocess_X(fe_params, X.copy(), X_test.copy())

# zero_model = DeepMultiLabelModel(nn_params, 'zero')

# y_zero = y_to_zero_class(y)
# zero_model.cv(X_p, y_zero, y0, n_split=7, run_name='tune_nn_1')

# zero_model.predict(X_test_p)<jupyter_output><empty_output><jupyter_text>Error class prediction model<jupyter_code># fe_params, nn_params, _, seed, _ = (
#     {'fe_cluster': {'enabled': True, 'n_clusters_c': 14, 'n_clusters_g': 39}, 'fe_stats': {'enabled': True}, 'pca': {'enabled': False}, 'scaler': 'quantile', 'shuffle_cols': True, 'variance_reduction': {'enabled': False}}, 
#     {'activation': ('elu', 'elu', 'elu', 'elu'), 'batch_norm': True, 
#      'dropout': (0.3, 0.3, 0.4, 0.3), 'epochs': 100, 'hid_layer': (512, 1024, 512, 2048), 
#      'init_non_scored_weights': False, 'label_smoothing': 0.0001380444271082826, 
#      'learning_rate': 0.4083831289327425, 'weight_norm': True,
# #      'class_weight': {0: 1, 1: 7, 2: 6, 3: 7, 4: 7, 5: 5}
#     }, 
#     {'zero_threshold': 0.9837308739197401}, 13, 2
# )
# seed_everything(seed)

# X_p, X_test_p = preprocess_X(fe_params, X.copy(), X_test.copy())

# error_model = DeepMultiLabelModel(nn_params, 'errors', verbose=0)

# y_error = y_to_error_classes(y)
# error_model.cv(X_p, y_error, y0, n_split=5)
# loss_df = error_model.errors(X_p, y_error)
# loss_df<jupyter_output><empty_output><jupyter_text>Calibration<jupyter_code># https://github.com/cerlymarco/MEDIUM_NoteBook/blob/master/NeuralNet_Calibration/NeuralNet_Calibration.ipynb
def fit_TemperatureCalibration(train_X_y, valid_X_y=None, epochs=100):
    
    ### inspired by: https://github.com/stellargraph/stellargraph/blob/develop/stellargraph/calibration.py ###
    T = tf.Variable(tf.ones(shape=(1,)))
    history = []
    early_stopping = False
    optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
    
    def cost(T, x, y):
        scaled_logits = tf.multiply(x=x, y=1.0 / T)
        cost_value = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits(logits=scaled_logits, labels=y)
        )

        return cost_value

    def grad(T, x, y):

        with tf.GradientTape() as tape:
            cost_value = cost(T, x, y)

        return cost_value, tape.gradient(cost_value, T)
    
    X_train, y_train = train_X_y
    if valid_X_y:
        X_valid, y_valid = valid_X_y
        early_stopping = True
    
    
    for epoch in range(epochs):
        train_cost, grads = grad(T, X_train, y_train)
        optimizer.apply_gradients(zip([grads], [T]))
        if early_stopping:
            val_cost = cost(T, X_valid, y_valid)
            if (len(history) > 0) and (val_cost > history[-1][1]):
                break
            else: 
                history.append([train_cost, val_cost, T.numpy()[0]])
        else:
            history.append([train_cost, T.numpy()[0]])

    history = np.asarray(history)
    temperature = history[-1, -1]
    
    return temperature


def calibrated_proba(logits, temperature):
    scaled_prediction = logits / temperature

    return np.exp(scaled_prediction) / np.sum(np.exp(scaled_prediction), axis=-1, keepdims=True)

def calibrate(model, X, y, y0, run_name=None, n_split=7, metric_fn=log_loss_metric, overfit=True):
    seed_everything(model.seed)

    ind_tr, ind_te = train_test_split(X, y, n_split=n_split)
    x_tr, y_tr, y0_tr = X.iloc[ind_tr, :].reset_index(drop=True), y.iloc[ind_tr, :].reset_index(drop=True), y0.iloc[ind_tr, :].reset_index(drop=True)
    x_te, y_te, y0_te = X.iloc[ind_te, :].reset_index(drop=True), y.iloc[ind_te, :].reset_index(drop=True), y0.iloc[ind_te, :].reset_index(drop=True)

    y_pred_te = model.predict(x_te)
    print(metric_fn(y_te, y_arr_to_df(y_pred_te, y_te.columns)))

    y_pred_tr = model.predict(x_tr)
    
    calib_temperature = fit_TemperatureCalibration((y_pred_tr, y_tr), (y_pred_te, y_te))
    print(calib_temperature)
    print(y_pred_te)
    y_pred_te = calibrated_proba(y_pred_te, calib_temperature)
    print(y_pred_te)

    y_pred_te = y_arr_to_df(y_pred_te, y_te.columns)
    return metric_fn(y_te, y_pred_te)

# calibrate(error_model, X_p.copy(), y_to_error_classes(y.copy()), y0.copy())<jupyter_output><empty_output><jupyter_text>Blender model<jupyter_code>!mkdir -p bundles
class ModelBlender(object):
    def __init__(self, deep_params, blend_params, seed=42, verbose=0, preprocess_params=None):
        self.deep_params = deep_params

        self.num_epochs = deep_params['epochs']
        self.blend_params = blend_params
        self.seed = seed
        self.preprocess_params = preprocess_params
        
        zero_params = blend_params.get('zero_params') or deep_params
        errors_params = blend_params.get('errors_params') or deep_params

        self.model_main = DeepMultiLabelModel(deep_params, model_name='main', seed=seed, verbose=verbose)
        self.model_zero = DeepMultiLabelModel(zero_params or deep_params, model_name='zero', seed=seed, verbose=verbose)
        self.model_errors = DeepMultiLabelModel(errors_params or deep_params, model_name='errors', seed=seed, verbose=verbose)
        self.calib_temperature = None
        self.classes = []
        self.metrics = {}

    @property
    def definition(self):
        return self.model_main.definition

    def predict(self, X):        
        y = self.model_main.predict(X)
        # Если выставлено, то принудительно ставим 0 везде, где модель предсказания 0 уверена
        zero_pred_threshold = self.blend_params.get('zero_threshold', 0)
        if zero_pred_threshold > 0:
            zero_preds = self.model_zero.predict(X)[:, 0]
            override_ind = zero_preds > zero_pred_threshold
            print('Override to zeros: {} rows'.format(len(override_ind[override_ind])))
            y[override_ind, :] = 0.

        if self.blend_params.get('use_error_class'):
            error_class_indices = [self.classes.index(e) for e in error_classes]
            y[:, error_class_indices] = self.model_errors.predict(X)
        return y
    
    def cv(self, X, y, y0, run_name=None, n_split=7, 
           metric_fn=log_loss_metric, overfit=True, 
           run_tags=None, return_pred=True
          ):
        seed_everything(self.seed)
        
        y_te = y.drop(columns=['drug_id'], errors='ignore')
        ycols = list(y_te.columns)
        self.classes = ycols
        
        preds = {}
        _, y_pred = self.model_main.cv(
            X, y, y0, 
            n_split=n_split, metric_fn=metric_fn, overfit=overfit,
            run_tags=run_tags, run_name=run_name, return_pred=True, max_score=0.019
        )
        if y_pred is None:
            if return_pred:
                return None, None
            return None

        self.metrics['initial'] = metric_fn(y_te, y_pred)
        preds['initial'] = y_pred

        print('INITIAL', self.metrics['initial'])
        zero_threshold = self.blend_params.get('zero_threshold', 0)
        if zero_threshold > 0:
            _, y_pred_zero = self.model_zero.cv(
                X, y_to_zero_class(y_te), y_to_zero_class(y0), 
                n_split=n_split, metric_fn=metric_fn, overfit=overfit,
                run_tags=run_tags, run_name=run_name, return_pred=True
            )
            zero_preds = y_pred_zero.iloc[:, 0]
            override_ind = zero_preds > zero_threshold
            print('Override to zeros: {} rows'.format(len(override_ind[override_ind])))
            y_pred.loc[override_ind, :] = 0.
            self.metrics['after_zero'] = metric_fn(y_te, y_pred)
            preds['after_zero'] = y_pred
            print('AFTER ZERO', self.metrics['after_zero'])
        
        if self.blend_params.get('use_error_class'):
            _, y_pred_errors = self.model_errors.cv(
                X, y_to_error_classes(y_te), y0, 
                n_split=n_split, metric_fn=metric_fn, overfit=overfit,
                run_tags=run_tags, run_name=run_name, return_pred=True
            )
            y_pred.loc[:, error_classes] = y_pred_errors.loc[:, error_classes]
            self.metrics['after_error'] = metric_fn(y_te, y_pred)
            preds['after_error'] = y_pred
            print('AFTER ERROR', self.metrics['after_error'])
        
        logloss_valid = metric_fn(y_te, y_pred)
        self.metrics['final'] = logloss_valid
        preds['final'] = y_pred

        if has_internet and run_name:
            bundle = {
                'seed': self.seed,
                'n_split': n_split,
                'p_min': p_min,
                'p_max': p_max,
                'preprocess_params': self.preprocess_params,
                'deep_params': self.deep_params,
                'blend_params': self.blend_params,
                'logloss_valid': logloss_valid,
                'metrics': self.metrics,
                'model_def': self.definition,
                'run_name': run_name,
                'run_tags': run_tags,
                'predictions': preds
            }
            bundle_path = os.path.join('bundles', f'{int(time())}.pickle')
            with open(bundle_path, 'wb') as fbundle:
                pickle.dump(bundle, fbundle)
            print(f'Write bundle: {bundle_path}')

            mlflow.set_experiment('Kaggle-MOA-Blend')
            with mlflow.start_run(run_name=run_name):
                mlflow.log_params(dict_flatten({
                    'seed': self.seed,
                    'n_split': n_split,
                    'p_min': p_min,
                    'p_max': p_max,
                    'preprocess_params': self.preprocess_params,
                    'deep_params': self.deep_params,
                    'blend_params': self.blend_params
                }))
                mlflow.log_metric(key="logloss_valid", value=logloss_valid)
                mlflow.log_metrics(self.metrics)
                tags = {
                    'model_def': self.definition,
                    'run': run_name,
                    'bundle_path': bundle_path
                }
                tags.update(run_tags)
                mlflow.set_tags(tags)
        if return_pred:
            return logloss_valid, y_pred
        return logloss_valid

evaluate_context = {}

def create_and_evaluate_model(
    args, models=None, predictions=None, predictions_cv=None,
    run_name='', n_split=7, verbose=0, overfit=False
):
    print(args)
    preprocess_params, deep_params, blend_params, seed, y_quantiles = args

    try:
        X_p, X_test_p, y_p, y0_p = preprocess_X(fe_params, X.copy(), X_test.copy(), y.copy(), y0.copy(), seed=seed)
    except Exception as e:
        print(e)
        return 0.19

    model = ModelBlender(deep_params, blend_params, seed=seed, verbose=verbose, preprocess_params=preprocess_params)
    evaluate_context['current_iter'] = {
        'model': model
    }
    logloss_valid, y_pred = model.cv(
        X_p, y_p, y0_p, n_split=n_split, overfit=overfit, 
        run_name=run_name, run_tags={'args': str(args)}, return_pred=True
    )
    if logloss_valid is None:
        return 0.19

    if models is not None:
        models.append(model)

    if predictions is not None:
        predictions.append(model.predict(X_test_p))
    
    if predictions_cv is not None:
        predictions_cv.append(y_pred)

    evaluate_context['last_iter'] = {
        'model': model,
        'logloss': logloss_valid
    }

    print(f'Final valid logloss: {logloss_valid}')
    return logloss_valid<jupyter_output><empty_output><jupyter_text>Manual tuning<jupyter_code>from copy import deepcopy
models_final = []

p_min = 0.0015
p_max = 0.9985
batch_size = 128
seed = 8
fold = fold_simple

baseline_num = 5
baseline_score = round(0.01562, 6)

fe_params = {
    'fe_cluster': {'enabled': False, 'n_clusters_c': 6, 'n_clusters_g': 44}, 
    'fe_stats': {'enabled': False}, 
    'pca': {'enabled': True, 'n_comp_cells': 50, 'n_comp_genes': 600}, 
    'scaler': 'quantile', 'shuffle_cols': True, 
    'variance_reduction': {'enabled': True, 'threshold': 0.8}
}
nn_params = {
    'batch_norm': True, 'weight_norm': True, 
    'activation': ['elu', 'elu', 'elu', 'elu'], 
    'dropout': [0.3, 0.3, 0.4, 0.3], 
    'hid_layer': [512,1024,512,2048], 
    'init_non_scored_weights': False, 
    'label_smoothing': 0.0015, 
    'learning_rate': 0.001, 'epochs': 25
}

baseline_conf = (
    fe_params, 
    nn_params, 
    {'zero_threshold': 0, "use_error_class": False}, 8, 1
)

new_conf = deepcopy(baseline_conf)
# new_conf[2]['use_error_class'] = True

new_score = create_and_evaluate_model(
    new_conf, models=models_final, 
    run_name=f'base_{baseline_num}', n_split=5, verbose=0
)

print(
    round(new_score - baseline_score, 6), 
    f'{100 * round(new_score/baseline_score, 3) - 100:.2f}%', 
    round(new_score, 6)
)<jupyter_output><empty_output><jupyter_text>Public models<jupyter_code>p_min = 1e-5
p_max = 1.-1e-5
batch_size = 128
fold = fold_simple<jupyter_output><empty_output><jupyter_text>keras NN +PCA with Label smoothing CV[0.01562] LB [0.01859]<jupyter_code># https://www.kaggle.com/riadalmadani/keras-nn-pca-with-label-smoothing

seed = 8

fe_params = {
    'fe_cluster': {'enabled': False, 'n_clusters_c': 6, 'n_clusters_g': 44}, 
    'fe_stats': {'enabled': False}, 
    'pca': {'enabled': True, 'n_comp_cells': 50, 'n_comp_genes': 600}, 
    'scaler': 'quantile', 'shuffle_cols': True, 
    'variance_reduction': {'enabled': True, 'threshold': 0.8}
}
X_p, X_test_p, y_p, y0_p = preprocess_X(fe_params, X.copy(), X_test.copy(), y.copy(), y0.copy(), seed=seed)
print(X_p.shape, 'should be: (, 1039)')

nn_params = {
    'batch_norm': True, 'weight_norm': True, 
    'activation': ['relu', 'relu', 'relu'], 
    'dropout': [0.2, 0.5, 0.2], 
    'hid_layer': [2048,1048,512], 
    'init_non_scored_weights': False, 
    'label_smoothing': 0.0015, 
    'learning_rate': 0.001, 'epochs': 500
}
nn_params = {
    'batch_norm': True, 'weight_norm': True, 
    'activation': ['elu', 'elu', 'elu', 'elu'], 
    'dropout': [0.3, 0.3, 0.4, 0.3], 
    'hid_layer': [512,1024,512,2048], 
    'init_non_scored_weights': False, 
    'label_smoothing': 0.0015, 
    'learning_rate': 0.001, 'epochs': 35
}
main_model = DeepMultiLabelModel(nn_params, 'main', seed=seed, verbose=0)
main_model.cv(X_p, y_p, y0_p, n_split=7, run_name='public_nn_1', overfit=True)
# Epoch 5/35 0.0180
# Epoch 10/35 0.0160<jupyter_output>(21948, 1043) should be: (, 1039)
[02:13], Fold 0: 0.016077665932851228
[02:40], Fold 1: 0.016139014846157457
[02:21], Fold 2: 0.016049516507951788
[02:16], Fold 3: 0.015945916239989495
[02:21], Fold 4: 0.016240827478692616
[02:29], Fold 5: 0.015985089542815856
[02:30], Fold 6: 0.015586100719255144
[02:30], Fold 7: 0.015697994209915803
[02:16], Fold 8: 0.015996964666843715
[02:08], Fold 9: 0.016317549712339566
[02:39], Fold 10: 0.015972715173830587
[02:16], Fold 11: 0.01589995101765791
Valid logloss: 0.015992442170691762<jupyter_text>Pytorch-RankGauss-PCA-NN CV [0.014572] LB [0.01839]<jupyter_code># https://www.kaggle.com/vbmokin/moa-pytorch-rankgauss-pca-nn-upgrade-3d-visual
seed = 0

fe_params = {
    'fe_cluster': {'enabled': False, 'n_clusters_c': 6, 'n_clusters_g': 44}, 
    'fe_stats': {'enabled': False}, 
    'pca': {'enabled': True, 'n_comp_cells': 60, 'n_comp_genes': 463}, 
    'scaler': 'quantile', 'shuffle_cols': True, 
    'variance_reduction': {'enabled': True, 'threshold': 0.9}
}

X_p, X_test_p, y_p, y0_p = preprocess_X(fe_params, X.copy(), X_test.copy(), y.copy(), y0.copy(), seed=seed)

print(X_p.shape, 'should be: (, 1015)')

nn_params = {
    'batch_norm': True, 'weight_norm': True, 
    'activation': ['leaky_relu', 'leaky_relu'], 
    'dropout': [0.25, 0.25], 
    'hid_layer': [1500, 1500], 
    'init_non_scored_weights': False, 
    'label_smoothing': 0,
    'learning_rate': 0.001, 'epochs': 25
}
main_model = DeepMultiLabelModel(nn_params, 'main', verbose=0, seed=seed)
main_model.cv(X_p, y_p, y0_p, n_split=7, run_name='public_nn_2', overfit=True)

# FOLD: 0, EPOCH: 5, valid_loss: 0.017283157035708426
# FOLD: 0, EPOCH: 10, valid_loss: 0.01737722285091877
# FOLD: 0, EPOCH: 24, valid_loss: 0.016081880070269106<jupyter_output>(21948, 1015) should be: (, 1015)
[01:54], Fold 0: 0.01641352227166248
[01:44], Fold 1: 0.016291257878713786<jupyter_text>MODEL1 CV [0.01562060391771847] LB [0.01833]<jupyter_code># https://www.kaggle.com/vikazrajpurohit/3-model-training-and-inference#1.-MODEL1-CV-[0.01562060391771847]-LB-[0.01833]
seed = 42

# TODO:
# Fine tune и полный transfer learning

fe_params = {
    'fe_cluster': {'enabled': True, 'n_clusters_c': 4, 'n_clusters_g': 22}, 
    'fe_stats': {'enabled': True}, 
    'pca': {'enabled': True, 'n_comp_cells': 50, 'n_comp_genes': 600, 'n_clusters': 5},
    'scaler': 'quantile', 'shuffle_cols': False, 
    'variance_reduction': {'enabled': True, 'threshold': 0.85}
}

y0_s = pd.concat([y, y0], axis=1)

X_p, X_test_p = preprocess_X(fe_params, X.copy(), X_test.copy(), seed=seed)
print(X_p.shape, 'should be: (, 1240)')

nn_params = {
    'batch_norm': True, 'weight_norm': 'output',
    'activation': ['leaky_relu', 'leaky_relu', 'leaky_relu', 'leaky_relu'], 
    'dropout': [0.5, 0.35, 0.3, 0.25], 
    'hid_layer': [1500, 1250, 1000, 750], 
    'init_non_scored_weights': 'ALL_TARGETS', 
    'label_smoothing': 0, 
    'learning_rate': 1e-3, 'epochs': 25
}
main_model = DeepMultiLabelModel(nn_params, 'main', verbose=0, seed=0)
main_model.cv(X_p, y, y0_s, n_split=7, run_name='tune_public_3', overfit=True)

# ALL_TARGETS:
    # SEED: 0, FOLD: 0, EPOCH: 5, train_loss: 0.013472, valid_loss: 0.009183
    # SEED: 0, FOLD: 0, EPOCH: 23, train_loss: 0.012016, valid_loss: 0.008624
# SCORED_ONLY:
    # SEED: 0, FOLD: 0, EPOCH: 5, train_loss: 0.019837, valid_loss: 0.016361<jupyter_output><empty_output><jupyter_text>Auto Tuning<jupyter_code>from hyperopt import fmin, tpe, hp, Trials, space_eval
from hyperopt.pyll.stochastic import sample as ho_sample

model_shapes = [
    # https://www.kaggle.com/sinamhd9/mechanisms-of-action-moa-tutorial/data
    [[1792, 1024, 2048], [0.65, 0.7, 0.6], ['relu', 'relu', 'relu']],
    [[1792, 1024, 2048], [0.65, 0.7, 0.6], ['selu','swish','swish']],
    [[1152, 1152, 2048], [0.7,0.7,0.3], ['selu', 'swish', 'swish']],
    [[1408, 1152, 1920], [0.7,0.65,0.45], ['selu','swish','swish']],
    [[1280,1152,1920], [0.7,0.55,0.3], ['selu','swish','swish']],
    [[896, 768, 2048], [0.7,0.7,0.3], ['selu','swish','elu']],
    [[1280, 768, 2048], [0.7,0.7,0.3], ['selu','swish','elu']],
    [[1152, 2048, 1280], [0.7,0.7,0.6], ['selu','swish','swish']],
    [[768,512,2048], [0.65,0.7,0.3], ['selu','swish','selu']],
    [[1664,1408,1280], [0.65,0.7,0.6], ['selu','swish','swish']],
    [[1408,1280,2048], [0.65, 0.65, 0.3], ['selu','swish','elu']],
    
    # https://www.kaggle.com/omniking1999/notebook-v3-0
    [[2048,1048,512], [0.2, 0.5, 0.2], ['relu', 'relu', 'relu']],
    [[512, 1024, 512, 2048], [0.3, 0.3, 0.4, 0.3], ['elu','elu','elu','elu']],
    [[1500, 1250, 1000, 750], [0.5, 0.35, 0.3, 0.25], ['leaky_relu', 'leaky_relu', 'leaky_relu', 'leaky_relu']]
]

seeds = [34, 9, 42, 11, 22, 8, 13, 25, 21, 29]

space_preprocess = [{
        'scaler': hp.choice('scaler', ['gauss', 'quantile', 'none', 'standard']),
        'fe_stats': hp.choice('fe_stats', [{
            "enabled": hp.choice('fe_stats_enabled', [True, False]),
        }]),
        'pca': hp.choice('pca', [{
            "enabled": hp.choice('pca_enabled1', [True]),
            "n_comp_genes": hp.uniformint('n_comp_genes', 300, 700),
            "n_comp_cells": hp.uniformint('n_comp_cells', 30, 90),
        }, {
            "enabled": hp.choice('pca_enabled2', [False]),
        }]),
        'fe_cluster': hp.choice('fe_cluster', [{
            "enabled": hp.choice('fe_cluster_enabled1', [False]),
        },
        {
            "enabled": hp.choice('fe_cluster_enabled2', [True]),
            "n_clusters_g": hp.uniformint('n_clusters_g', 15, 45),
            "n_clusters_c": hp.uniformint('n_clusters_c', 5, 15),
        }]),
        'variance_reduction': hp.choice('variance_reduction', [{
            "enabled": hp.choice('variance_reduction_enabled1', [True]),
            "threshold": hp.uniform('threshold', 0.7, 0.9)
        }, {
            "enabled": hp.choice('variance_reduction_enabled2', [False]),
        }]),
        'shuffle_cols': hp.choice('shuffle_cols', [True, False]),
    }]

space_deep = [{
        "hid_layer": hp.choice(f'hid_layer_{i}', [model_shape[0]]),
        'activation': hp.choice(f'activation_{i}', [model_shape[2]]),
        'dropout': hp.choice(f'dropout_{i}', [model_shape[1]]),
        'learning_rate': hp.uniform(f'learning_rate_{i}', 0.001, 0.5),
        'label_smoothing': hp.uniform(f'label_smoothing{i}', 0.0001, 0.003),
        'init_non_scored_weights': hp.choice(f'init_non_scored_weights_{i}', ['ALL_TARGETS', 'ONLY_NON_SCORED', False]),
        'batch_norm': hp.choice(f'batch_norm_{i}', [True, False]),
        'weight_norm': hp.choice(f'weight_norm_{i}', [True, False]),
        'epochs': hp.choice(f'epochs_{i}', [50, 100, 200])
    } for i, model_shape in enumerate(model_shapes)
]<jupyter_output><empty_output><jupyter_text>Error class<jupyter_code>error_class_space = [
    hp.choice('preprocess', space_preprocess),
    hp.choice('deep_params', space_deep),
    hp.choice('seed', seeds)
]

def evaluate_error_class(args, run_name=None, n_split=7):
    print(args)
    fe_params, nn_params, seed = args
    seed_everything(seed)

    X_p, X_test_p, y_p, y0_p = preprocess_X(fe_params, X.copy(), X_test.copy(), y.copy(), y0.copy(), seed=seed)

    error_model = DeepMultiLabelModel(nn_params, 'errors', verbose=0)

    y_error = y_to_error_classes(y_p)
    return error_model.cv(
        X_p, y_error, y0_p, n_split=n_split, run_name=run_name,
        run_tags={"args": str(args)}
    )

trials_version = 'error_class'
run_name = f'tune_{trials_version}'
space = error_class_space
score_func = partial(evaluate_error_class, run_name=run_name, n_split=5)

ho_sample(space)<jupyter_output><empty_output><jupyter_text>Zero class<jupyter_code>zero_class_space = [
    hp.choice('preprocess', space_preprocess),
    hp.choice('deep_params', space_deep),
    hp.choice('seed', seeds)
]

def evaluate_zero_class(args, run_name=None, run_tags=None, n_split=7):
    print(args)
    fe_params, nn_params, seed = args
    seed_everything(seed)

    X_p, X_test_p, y_p, y0_p = preprocess_X(fe_params, X.copy(), X_test.copy(), y.copy(), y0.copy(), seed=seed)

    error_model = DeepMultiLabelModel(nn_params, 'zero', verbose=0)

    y_error = y_to_zero_class(y_p)
    return error_model.cv(
        X_p, y_error, y_to_zero_class(y0_p), n_split=n_split, 
        run_name=run_name, run_tags={"args": str(args)}
    )

trials_version = 'zero_class'
run_name = f'tune_{trials_version}'
space = zero_class_space
score_func = partial(evaluate_zero_class, run_name=run_name, n_split=5)

ho_sample(space)<jupyter_output><empty_output><jupyter_text>Blender<jupyter_code>blend_space = [
    hp.choice('preprocess', space_preprocess),
    hp.choice('deep_params', space_deep),
    hp.choice('blend_params', [
    {
        'zero_threshold': hp.uniform('zero_threshold1', 0.95, 0.999),
        'use_error_class': hp.choice('use_error_class1', [True, False]),
    },
    {
        'zero_threshold': hp.choice('zero_threshold2', [0]),
        'use_error_class': hp.choice('use_error_class2', [True, False]),
    }]),
    hp.choice('seed', seeds),
    hp.choice('y_quantiles', [1, 2]),
]

trials_version = 'blender2'
run_name = f'tune_{trials_version}'
space = blend_space
score_func = partial(create_and_evaluate_model, models=None, run_name=run_name, n_split=7)

ho_sample(space)<jupyter_output><empty_output><jupyter_text>Run!<jupyter_code>!mkdir -p trials/
start_secs = time()
trials = Trials()

fold = fold_simple

trials_file = "trials/trial_{}.hp".format(trials_version)
if isfile(trials_file):
    print(f'load trials from: {trials_file}')
    trials = pickle.load(open(trials_file, 'rb'))

EPOCHS = None

for i in range(60):
    sug = tpe.suggest
    if i % 3 == 0:
        sug = tpe.rand.suggest
    best = fmin(
        fn=score_func,
        space=space,
        algo=sug,
        max_evals=2 * (i + 1),
        trials=trials
    )
    pickle.dump(trials, open(trials_file, "wb"))<jupyter_output>({'fe_cluster': {'enabled': True, 'n_clusters_c': 5, 'n_clusters_g': 19}, 'fe_stats': {'enabled': False}, 'pca': {'enabled': False}, 'scaler': 'none', 'shuffle_cols': True, 'variance_reduction': {'enabled': True, 'threshold': 0.7234634592150213}}, {'activation': ('selu', 'swish', 'selu'), 'batch_norm': False, 'dropout': (0.65, 0.7, 0.3), 'epochs': 100, 'hid_layer': (768, 512, 2048), 'init_non_scored_weights': 'ALL_TARGETS', 'label_smoothing': 0.0006978288035937164, 'learning_rate': 0.3178535920260323, 'weight_norm': True}, {'use_error_class': True, 'zero_threshold': 0.985059066480174}, 22, 1)
[02:33], Fold 0: 0.019598324077761204                
break cv execution 0.019598324077761204 > 0.019      
({'fe_cluster': {'enabled': True, 'n_clusters_c': 15, 'n_clusters_g': 43}, 'fe_stats': {'enabled': True}, 'pca': {'enabled': True, 'n_comp_cells': 74, 'n_comp_genes': 557}, 'scaler': 'quantile', 'shuffle_cols': True, 'variance_reduction': {'enabled': True, 'threshold': 0.7189569644347072}}, [...]<jupyter_text>Final prediction<jupyter_code># Сюда записываем все доступные предсказания для блендинга
from copy import deepcopy

fold = fold_simple

models_final = []
predictions_final = []
baseline_num = 2
EPOCHS = None
SPLITS = 12

final_conf = [
    # http://datadigger.ru:5000/#/experiments/3/runs/4525174c3e574ac68426e02ab0f0ee3f
#     ({'fe_cluster': {'enabled': True, 'n_clusters_c': 14, 'n_clusters_g': 39}, 'fe_stats': {'enabled': True}, 'pca': {'enabled': False}, 'scaler': 'quantile', 'shuffle_cols': True, 'variance_reduction': {'enabled': False}}, {'activation': ('elu', 'elu', 'elu', 'elu'), 'batch_norm': True, 'dropout': (0.3, 0.3, 0.4, 0.3), 'epochs': 100, 'hid_layer': (512, 1024, 512, 2048), 'init_non_scored_weights': False, 'label_smoothing': 0.0001380444271082826, 'learning_rate': 0.4083831289327425, 'weight_norm': True}, {'use_error_class': True, 'zero_threshold': 0.985}, 13, 2),
    # http://datadigger.ru:5000/#/experiments/3/runs/0b83844bc5bd44b2a9665fe6bd9feee4
#     ({'fe_cluster': {'enabled': True, 'n_clusters_c': 11, 'n_clusters_g': 41}, 'fe_stats': {'enabled': False}, 'pca': {'enabled': False}, 'scaler': 'quantile', 'shuffle_cols': True, 'variance_reduction': {'enabled': False}}, {'activation': ('elu', 'elu', 'elu', 'elu'), 'batch_norm': True, 'dropout': (0.3, 0.3, 0.4, 0.3), 'epochs': 100, 'hid_layer': (512, 1024, 512, 2048), 'init_non_scored_weights': False, 'label_smoothing': 0.00010848528437984486, 'learning_rate': 0.40431872111746137, 'weight_norm': True}, {'use_error_class': True, 'zero_threshold': 0.985}, 13, 1),
    # http://datadigger.ru:5000/#/experiments/0/runs/82cc5898a2b54ce5832d93f6a1afd445
    ({'fe_cluster': {'enabled': True, 'n_clusters_c': 6, 'n_clusters_g': 44}, 'fe_stats': {'enabled': False}, 'pca': {'enabled': False}, 'scaler': 'none', 'shuffle_cols': True, 'variance_reduction': {'enabled': True, 'threshold': 0.9585049261745544}}, {'activation': ('selu', 'swish', 'swish'), 'batch_norm': True, 'dropout': (0.7, 0.7, 0.3), 'epochs': 100, 'hid_layer': (1152, 1152, 2048), 'init_non_scored_weights': False, 'label_smoothing': 0.0007000000000000001, 'learning_rate': 0.016, 'weight_norm': True}, {'use_error_class': True, 'zero_threshold': 0.985}, 29, 1),
    
    (
    {
        'fe_cluster': {'enabled': False, 'n_clusters_c': 6, 'n_clusters_g': 44}, 
        'fe_stats': {'enabled': False}, 
        'pca': {'enabled': True, 'n_comp_cells': 50, 'n_comp_genes': 600}, 
        'scaler': 'quantile', 'shuffle_cols': True, 
        'variance_reduction': {'enabled': True, 'threshold': 0.8}
    },
    {
        'batch_norm': True, 'weight_norm': True, 
        'activation': ['elu', 'elu', 'elu', 'elu'], 
        'dropout': [0.3, 0.3, 0.4, 0.3], 
        'hid_layer': [512,1024,512,2048], 
        'init_non_scored_weights': False, 
        'label_smoothing': 0.0015, 
        'learning_rate': 0.001, 'epochs': 25
    }, 
        {
            'use_error_class': True, 'zero_threshold': 0.985
        },
        8, 1
    ),
    # public 1
    (
        {
            'fe_cluster': {'enabled': False, 'n_clusters_c': 6, 'n_clusters_g': 44}, 
            'fe_stats': {'enabled': False}, 
            'pca': {'enabled': True, 'n_comp_cells': 50, 'n_comp_genes': 600}, 
            'scaler': 'quantile', 'shuffle_cols': True, 
            'variance_reduction': {'enabled': True, 'threshold': 0.8}
        },
        {
            'batch_norm': True, 'weight_norm': True, 
            'activation': ['elu', 'elu', 'elu', 'elu'], 
            'dropout': [0.3, 0.3, 0.4, 0.3], 
            'hid_layer': [512,1024,512,2048], 
            'init_non_scored_weights': False, 
            'label_smoothing': 0.0015, 
            'learning_rate': 0.001, 'epochs': 35
        },
        {
            'use_error_class': True, 'zero_threshold': 0.985
        },
        8, 1
    ),
    # best mlflow
    (
        {'fe_cluster': {'enabled': False}, 'fe_stats': {'enabled': False}, 'pca': {'enabled': False}, 'scaler': 'gauss', 'shuffle_cols': False, 'variance_reduction': {'enabled': False}}, 
        {'activation': ('selu', 'swish', 'swish'), 'batch_norm': False, 'dropout': (0.65, 0.7, 0.6), 'epochs': 200,
         'hid_layer': (1664, 1408, 1280), 'init_non_scored_weights': True, 'label_smoothing': 0.0001087446974391731, 'learning_rate': 0.04150188777502751, 'weight_norm': True}, 
        {
            'use_error_class': True, 'zero_threshold': 0.985
        }, 
        34, 1
    ),
    # public 2
    ({
        'fe_cluster': {'enabled': False, 'n_clusters_c': 6, 'n_clusters_g': 44}, 
        'fe_stats': {'enabled': False}, 
        'pca': {'enabled': True, 'n_comp_cells': 60, 'n_comp_genes': 463}, 
        'scaler': 'quantile', 'shuffle_cols': True, 
        'variance_reduction': {'enabled': True, 'threshold': 0.9}
    },
    {
        'batch_norm': True, 'weight_norm': True, 
        'activation': ['leaky_relu', 'leaky_relu'], 
        'dropout': [0.25, 0.25], 
        'hid_layer': [1500, 1500], 
        'init_non_scored_weights': False, 
        'label_smoothing': 0,
        'learning_rate': 0.001, 'epochs': 25
    },
    {
        'use_error_class': True, 'zero_threshold': 0.985
    },
    0, 1
    ),
    # public 3
    ({
        'fe_cluster': {'enabled': True, 'n_clusters_c': 4, 'n_clusters_g': 22}, 
        'fe_stats': {'enabled': True}, 
        'pca': {'enabled': True, 'n_comp_cells': 50, 'n_comp_genes': 600, 'n_clusters': 5},
        'scaler': 'quantile', 'shuffle_cols': False, 
        'variance_reduction': {'enabled': True, 'threshold': 0.85}
    },
    {
        'batch_norm': True, 'weight_norm': 'output',
        'activation': ['leaky_relu', 'leaky_relu', 'leaky_relu', 'leaky_relu'], 
        'dropout': [0.5, 0.35, 0.3, 0.25], 
        'hid_layer': [1500, 1250, 1000, 750], 
        'init_non_scored_weights': True, 
        'label_smoothing': 0, 
        'learning_rate': 1e-3, 'epochs': 25
    },
    {
        'use_error_class': True, 'zero_threshold': 0.985
    },
    42, 1
    ),
]

cv_preds = []
for conf in final_conf:
    conf2 = deepcopy(conf)
    new_score = create_and_evaluate_model(
        conf2,
        models=models_final, 
        predictions=predictions_final,
        predictions_cv=cv_preds,
        n_split=SPLITS,
        run_name=f'base_{baseline_num}'
    )

print('\nFinal results:')
# http://datadigger.ru:5000/#/experiments/3/runs/69b1f266675941378e47d747fd5c11fc
print('Should be lower than', 0.01709)

yres = y.drop(columns=['drug_id'], errors='ignore')
ycols = yres.columns
cv_res = []
for i, cv_pred in enumerate(cv_preds):
    print(f'Logloss {i}:', log_loss_metric(yres, cv_pred))
    cv_res.append(cv_pred.values)

final_pred = y_arr_to_df(np.mean(cv_res, axis=0), ycols)
print(f'CV:', log_loss_metric(yres, final_pred))

# Формируем предсказания на основе множества предсказаний
# Сюда записываем submission предсказания
df_sample.loc[:, ycols] = 0
df_sample.loc[:, ycols] = np.mean(predictions_final, axis=0)
# У ctl_vehicle все классы - 0, поэтому просто зануляем
# Правильно ли?
df_sample.loc[ind_te, ycols] = 0
display(df_sample.head())<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>df_sample.to_csv('submission.csv', index=False)<jupyter_output><empty_output><jupyter_text>Error analysis<jupyter_code># Количество ошибок, когда должно быть везде 0, а на самом деле - нет
# Количество ошибок, когда должно что-то быть, а на самом деле везде 0
# Для уменьшения этих ошибок можно применять lgb_zero

res = y.copy()
res.loc[:, y.columns] = 0

# model = models_final[-1]
# preprocess_params, _, _, seed, _ = baseline_conf[-1]

model = error_model
preprocess_params = fe_params

seed_everything(seed)

_, te = train_test_split(X, y, n_split=5)
X_p, _, _, _ = preprocess_X(fe_params, X.copy(), X_test.copy(), y.copy(), y0.copy(), seed=seed)

x_val = X_p.astype('float64').values[te]

res.loc[te, y.columns] = model.predict(x_val)

y_pred = res.loc[te, y.columns]
y_true = y.loc[te, y.columns]
non_zeros = y_true[(y_true.T != 0).any()].index
all_zeros = y_true[~(y_true.T != 0).any()].index

clip_p_min = 1e-5
clip_p_max = 1 - 1e-5

y_pred_clip = np.clip(y_pred, clip_p_min, clip_p_max)

print('Logloss:', log_loss_metric(y_true, y_pred))
print('Logloss all zeros:', log_loss_metric(y_true.loc[all_zeros, :], y_pred.loc[all_zeros, :]))
print('Logloss non zeros:', log_loss_metric(y_true.loc[non_zeros, :], y_pred.loc[non_zeros, :]))
print('Logloss clip:', log_loss_metric(y_true, y_pred_clip))
print('Logloss all zeros clip:', log_loss_metric(y_true.loc[all_zeros, :], y_pred_clip.loc[all_zeros, :]))
print('Logloss non zeros clip:', log_loss_metric(y_true.loc[non_zeros, :], y_pred_clip.loc[non_zeros, :]))

losses = []
for i in range(y_true.shape[1]):
    y_true_cl = y_true.iloc[:,i]
    y_pred_cl = y_pred.iloc[:,i]
    losses.append({
        "index": i,
        "class": y_true_cl.name,
        'true_0': len(y_true_cl[y_true_cl == 0]),
        'true_1': len(y_true_cl[y_true_cl == 1]),
        "loss": log_loss(y_true_cl.values, y_pred_cl.values, labels=[0, 1]),
        'pred_hist_0': y_pred_cl[y_pred_cl <= 0.5].round(1).value_counts().sort_index().reset_index().values,
        'pred_hist_1': y_pred_cl[y_pred_cl > 0.5].round(1).value_counts().sort_index().reset_index().values,
    })

loss_df = pd.DataFrame(losses).set_index(['index', 'class']).sort_values('loss', ascending=False)
display(loss_df)<jupyter_output>Logloss: 0.013384161126035564
Logloss all zeros: 0.0026687569508067285
Logloss non zeros: 0.02042709727321972
Logloss clip: 0.013384161126035564
Logloss all zeros clip: 0.0026687569508067285
Logloss non zeros clip: 0.02042709727321972
{'fraction_non_alphanumeric': 0.15275850872273225, 'fraction_numerical': 0.04706733181804764, 'mean_word_length': 3.38033359193173, 'pattern_counts': {'":': 22, '<': 91, '<?xml version=': 0, '>': 103, 'https://': 12, 'lorem ipsum': 0, 'www.': 9, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '23', 'dir': 'jupyter-structured-clean-dedup', 'id': '395411', 'license': 'Unlicense', 'n_tokens_mistral': 27934, 'n_tokens_neox': 25628, 'n_words': 4557, 'path': 'moa/kaggle-moa.ipynb', 'repo_name': 'bataeves/kaggle', 'size': '67754'}