<jupyter_start><jupyter_text>Load Dataset<jupyter_code>train_data = pd.read_csv('../input/tabular-playground-series-nov-2021/train.csv',index_col='id')
test_data = pd.read_csv('../input/tabular-playground-series-nov-2021/test.csv',index_col='id')
sample_sub_data = pd.read_csv('../input/tabular-playground-series-nov-2021/sample_submission.csv',index_col='id')
column_y = train_data.columns.difference(test_data.columns)[0]
x = train_data.drop(columns=column_y)
y = train_data[column_y]

x_test = test_data.iloc[:,:]<jupyter_output><empty_output><jupyter_text>Split Dataset<jupyter_code># Check NA
missing_val = x.isnull().sum()
print(missing_val[missing_val > 0])

from sklearn.model_selection import train_test_split 

# x, x_val, y, y_val = train_test_split(x, y, train_size = 0.85, random_state = RANDOM_STATE)<jupyter_output>Series([], dtype: int64)<jupyter_text>Scaler transformer<jupyter_code>from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.pipeline import make_pipeline
from sklearn.compose import make_column_transformer

transformer_all_cols = make_pipeline(
    StandardScaler(),
    MinMaxScaler(feature_range=(0, 1))
)

preprocessor = make_column_transformer(
    (transformer_all_cols, x.columns[:]),
)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>import keras 
from tensorflow.keras import layers
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau

def load_model(name:str):
    
    early_stopping = EarlyStopping(
        patience=20,
        min_delta=0,
        monitor='val_loss',
        restore_best_weights=True,
        verbose=0,
        mode='min', 
        baseline=None,
    )
    plateau = ReduceLROnPlateau(
            monitor='val_loss', 
            factor=0.2, 
            patience=7, 
            verbose=0,
            mode='min')

# -----------------------------------------------------------------
# Model 

    model = keras.Sequential([
    layers.BatchNormalization(),
    layers.Dense(128, activation=ACTIVATION, input_shape = [x.shape[1]]),
    layers.Dropout(rate = 0.5),
    layers.BatchNormalization(),
    layers.Dense(64, activation=ACTIVATION),
    layers.Dropout(rate = 0.5),
    layers.BatchNormalization(),
    layers.Dense(32, activation=ACTIVATION), 
    layers.Dropout(rate = 0.5),
    layers.Dense(1, activation = 'sigmoid'),
    ])

# -----------------------------------------------------------------

    model.compile(
        optimizer= 'adam',
        loss='binary_crossentropy',
        metrics=['AUC'],
    )
    
    return model, early_stopping, plateau<jupyter_output><empty_output><jupyter_text>Kfold<jupyter_code>from sklearn.metrics import roc_auc_score, log_loss
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.preprocessing import StandardScaler, RobustScaler, PowerTransformer

preds_valid_f = {}
preds_test = []
total_auc = []
f_scores = []

kf = StratifiedKFold(n_splits=FOLDS,random_state=RANDOM_STATE,shuffle=True)


for fold,(train_index, valid_index) in enumerate(kf.split(x,y)):

    X_train, X_valid = x.loc[train_index], x.loc[valid_index]
    y_train, y_valid = y.loc[train_index], y.loc[valid_index]

    #   --------------------------------------------------------  
    # Preprocessing
    index_valid  = X_valid.index.tolist()
    test  = test_data.copy()
    
    X_train = preprocessor.fit_transform(X_train)
    X_valid = preprocessor.transform(X_valid)
    test = preprocessor.transform(test)
    
    #  ----------------------------------------------------------    
    # Model
    
    model, early_stopping, plateau  = load_model('version8')

    history = model.fit(  X_train, y_train,
                validation_data = (X_valid, y_valid),
                batch_size = BATCH_SIZE, 
                epochs = EPOCHS,
                callbacks = [early_stopping, plateau],
                shuffle = True,
                verbose = 0
              )

    #  ----------------------------------------------------------
    #  oof
    preds_valid = model.predict(X_valid).reshape(1,-1)[0] 
    
    #  ----------------------------------------------------------
    #  test  predictions
    preds_test.append(model.predict(test).reshape(1,-1)[0])
    
    #  ----------------------------------------------------------
    #  Saving  scores to plot the end  
    scores = pd.DataFrame(history.history)
    scores['folds'] = fold
    
    if fold == 0:
        f_scores = scores 
    else: 
        f_scores = pd.concat([f_scores, scores], axis  = 0)
        
    #  ----------------------------------------------------------
    #  concatenating valid preds
    preds_valid_f.update(dict(zip(index_valid, preds_valid)))

    # Getting score for a fold model
    fold_auc = roc_auc_score(y_valid, preds_valid)
    print(f"Fold {fold} roc_auc_score: {fold_auc}")

    # Total auc
    total_auc.append(fold_auc)

print(f"mean roc_auc_score: {np.mean(total_auc)}, std: {np.std(total_auc)}")<jupyter_output>User settings:

   KMP_AFFINITY=granularity=fine,verbose,compact,1,0
   KMP_BLOCKTIME=0
   KMP_DUPLICATE_LIB_OK=True
   KMP_INIT_AT_FORK=FALSE
   KMP_SETTINGS=1
   KMP_WARNINGS=0

Effective settings:

   KMP_ABORT_DELAY=0
   KMP_ADAPTIVE_LOCK_PROPS='1,1024'
   KMP_ALIGN_ALLOC=64
   KMP_ALL_THREADPRIVATE=128
   KMP_ATOMIC_MODE=2
   KMP_BLOCKTIME=0
   KMP_CPUINFO_FILE: value is not defined
   KMP_DETERMINISTIC_REDUCTION=false
   KMP_DEVICE_THREAD_LIMIT=2147483647
   KMP_DISP_NUM_BUFFERS=7
   KMP_DUPLICATE_LIB_OK=true
   KMP_ENABLE_TASK_THROTTLING=true
   KMP_FORCE_REDUCTION: value is not defined
   KMP_FOREIGN_THREADS_THREADPRIVATE=true
   KMP_FORKJOIN_BARRIER='2,2'
   KMP_FORKJOIN_BARRIER_PATTERN='hyper,hyper'
   KMP_GTID_MODE=3
   KMP_HANDLE_SIGNALS=false
   KMP_HOT_TEAMS_MAX_LEVEL=1
   KMP_HOT_TEAMS_MODE=0
   KMP_INIT_AT_FORK=true
   KMP_LIBRARY=throughput
   KMP_LOCK_KIND=queuing
   KMP_MALLOC_POOL_INCR=1M
   KMP_NUM_LOCKS_IN_BLOCK=1
   KMP_PLAIN_BARRIER='2,2'
   KMP_PLAIN_BARRIER_P[...]<jupyter_text>AUC<jupyter_code>import matplotlib.pyplot as plt

for fold in range(f_scores['folds'].nunique()):
    history_f = f_scores[f_scores['folds'] == fold]

    fig, ax = plt.subplots(1, 2, tight_layout=True, figsize=(14,4))
    fig.suptitle('Fold : '+str(fold), fontsize=14)
        
    plt.subplot(1,2,1)
    plt.plot(history_f.loc[:, ['loss', 'val_loss']], label= ['loss', 'val_loss'])
    plt.legend(fontsize=15)
    plt.grid()
    
    plt.subplot(1,2,2)
    plt.plot(history_f.loc[:, ['auc', 'val_auc']],label= ['auc', 'val_auc'])
    plt.legend(fontsize=15)
    plt.grid()
    
    print("Validation Loss: {:0.4f}".format(history_f['val_loss'].min()));<jupyter_output>Validation Loss: 0.5668
Validation Loss: 0.5684
Validation Loss: 0.5651
Validation Loss: 0.5658
Validation Loss: 0.5641<jupyter_text>Submission<jupyter_code>sub = pd.read_csv("../input/tabular-playground-series-nov-2021/sample_submission.csv")
sub['target'] = np.mean(preds_test, axis = 0)
sub.to_csv('submission.csv', index=False)
sub.head()
# sub['target'] = np.mean(preds_test, axis = 0)
# sub<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17890878330748625, 'fraction_numerical': 0.02001973776963203, 'mean_word_length': 2.895661724327293, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '474829', 'license': 'MIT', 'n_tokens_mistral': 2619, 'n_tokens_neox': 2508, 'n_words': 352, 'path': 'Classification/kerassimple.ipynb', 'repo_name': 'furyhawk/kaggle_practice', 'size': '7093'}