<jupyter_start><jupyter_text>TRAIN<jupyter_code>num_feature = len(FEATURE)
hidden = int(len(FEATURE) * 2)
#train test split for optuna-study
train_x, valid_x, train_y, valid_y = train_test_split(
    train[FEATURE].values, train[TARGET_COL].values, random_state = RANDOM_STATE, 
    stratify = train[TARGET_COL].values, test_size = .75
)

gc.collect()
mask_train = (train[REDUCED_FOLD_NAME] == 0)
mask_test = (train[REDUCED_FOLD_NAME] == 1)


train_x, train_y = train.loc[mask_train, FEATURE].values, train.loc[mask_train, TARGET_COL].values
valid_x, valid_y = train.loc[mask_test, FEATURE].values, train.loc[mask_test, TARGET_COL].values
class Model_list(nn.Module):
    def __init__(self, output_dims: List[int], dropout_list: List[float], num_features):
        super().__init__()
        
        layers: List[nn.Module] = []

        input_dim: int = num_features
        for i, output_dim in enumerate(output_dims):
            layers.append(nn.BatchNorm1d(input_dim))
            layers.append(nn.Linear(input_dim, output_dim))
            layers.append(nn.GELU())
            layers.append(nn.Dropout(dropout_list[i]))
            input_dim = output_dim
        
        layers.append(nn.BatchNorm1d(input_dim))
        layers.append(nn.Linear(input_dim, 1))
    
        self.layers: nn.Module = nn.Sequential(*layers)
    def forward(self, data: torch.Tensor) -> torch.Tensor:
        logits = self.layers(data)
        return logits
def run_training_opt(train, valid, fold, model, batch_size, epochs, 
                 seed, early_stop_step, early_stop, device, learning_rate, weight_decay):

    assert isinstance(train, list) & isinstance(valid, list) & (len(train) == 2) & (len(valid) == 2)
    
    seed_everything(seed)
            
    x_train, y_train  = train
    x_valid, y_valid =  valid
    
    train_dataset = TabularDataset(x_train, y_train)
    valid_dataset = TabularDataset(x_valid, y_valid)
    
    trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
    validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
        
    model.to(device)
    
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
    
    scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, max_lr=learning_rate*10, epochs=epochs, steps_per_epoch=len(trainloader))
    
    criterion = nn.BCEWithLogitsLoss()
    
    early_step = 0
    
    best_loss = np.inf
    best_auc = -np.inf
    
    for epoch in range(epochs):
        
        train_loss = train_fn(model, optimizer, scheduler, criterion, trainloader, device)
        valid_loss, valid_preds = valid_fn(model, criterion, validloader, device)

        valid_auc = roc_auc_score(y_valid, valid_preds)
                
        if valid_auc > best_auc:
            
            best_auc = valid_auc
            best_pred = valid_preds
                        
        elif(early_stop == True):
            
            early_step += 1
            if (early_step >= early_stop_step):
                break
        
    return best_auc

def objective(trial: optuna.trial.Trial) -> float:

    # We optimize the number of layers, hidden units in each layer and dropouts.
    n_layers = trial.suggest_int("n_layers", 1, 6)
    
    dropout_list = [
        trial.suggest_float("dropout_{}".format(i), 0, 0.5) for i in range(n_layers)
    ]
    
    output_dims = [
        trial.suggest_int("n_units_l{}".format(i), 4, 2000) for i in range(n_layers)
    ]

    model_ff = Model_list(output_dims, dropout_list, num_feature)
    
    auc = run_training_opt([train_x, train_y], [valid_x, valid_y], fold = -1, model = model_ff, batch_size = BATCH_SIZE, epochs = EPOCHS, 
                                     seed = RANDOM_STATE, early_stop_step = 3, early_stop = EARLY_STOP, 
                                     device = DEVICE, learning_rate = LEARNING_RATE, weight_decay = WEIGHT_DECAY)
    
    return auc
study = optuna.create_study(direction="maximize")
study.optimize(objective, timeout=30000, show_progress_bar = True)<jupyter_output>[32m[I 2021-10-17 06:36:01,416][0m A new study created in memory with name: no-name-c836d2fe-26ef-4482-89b7-d0ff077ae4e5[0m
/opt/conda/lib/python3.7/site-packages/optuna/progress_bar.py:47: ExperimentalWarning: Progress bar is experimental (supported from v1.2.0). The interface can change in the future.
  self._init_valid()<jupyter_text>Save result<jupyter_code>best_score = study.best_trial.values
print(best_score)
final_params = study.best_trial.params
print(final_params)
with open("final_nn_param.pkl", "wb") as file_name:
    pickle.dump(final_params, file_name)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10676532769556026, 'fraction_numerical': 0.01649048625792812, 'mean_word_length': 3.1391076115485563, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '132436', 'license': 'MIT', 'n_tokens_mistral': 1674, 'n_tokens_neox': 1590, 'n_words': 297, 'path': 'Tabular Playground Series - Oct 2021/optuna/optuna-nn-py.ipynb', 'repo_name': 'DavideStenner/Kaggle', 'size': '4730'}