<jupyter_start><jupyter_text>TPE (Tree-structured Parzen Estimator) and Sequential Least Squares Programming (SLSQP)https://optuna.readthedocs.io/en/stable/reference/generated/optuna.samplers.TPESampler.htmloptuna.samplers.TPESamplerhttps://docs.scipy.org/doc/scipy/reference/optimize.minimize-slsqp.html<jupyter_code>def run_inference_scripts(submission, weights=None, target_weights=None):
    for i, (script, oof_filename, output_filename, weight) in enumerate(model_list):
        print(f"Generating submission file from {script} ......")
        infer_start = time.time()
        !python {model_path}/{script}
        infer_elapsed = time.time() - infer_start
        print(f"Time spent on inference: {infer_elapsed/60:.2f} minutes.")

        model_submit = pd.read_csv(output_filename, engine='c')
        print(model_submit.head(5))
        print(model_submit.shape)

        if target_weights is not None:
            for j, target in enumerate(train_classes):
                print(f"Blending {script} for {target} with weight: {optimized_target_weights[j][i]} ......")
                submission.iloc[:, j+1] += model_submit.iloc[:, j+1] * optimized_target_weights[j][i]
        elif weights is None:
            print(f"Blending {script} with weight: {weight} ......")
            submission.iloc[:, 1:] += weight * model_submit.iloc[:, 1:]
        else:
            print(f"Blending {script} with weight: {weights[i]} ......")
            submission.iloc[:, 1:] += weights[i] * model_submit.iloc[:, 1:]

    return submission
total_start = time.time()

if not search_mode and run_submit_script:
    if method == "scipy_per_target":
        weights_path = glob.glob(f'{model_path}/{study_name}_*.pkl')[0]
        print(f"Loading target-wise optimized weights from {weights_path} ......")
        optimized_target_weights = load_pickle(weights_path)

        # For 206 target weights
        submission = run_inference_scripts(
            submission, target_weights=optimized_target_weights)
    else:
        submission = run_inference_scripts(submission)

elif search_mode and method == "CV":
    y_true = non_control_group_train_labels[train_classes].values

    all_oof = np.zeros(
        (len(model_list), non_control_group_train_labels.shape[0], 206))
    blend_oof = np.zeros((non_control_group_train_labels.shape[0], 206))
    print(all_oof.shape)
    for i, (script, oof_filename, output_filename,
            weight) in enumerate(model_list):
        print(f"Loading OOF from {oof_filename} ......")
        oof = np.load(f"{dataset_folder}/{oof_filename}")

        if oof.shape[0] == 23814:
            oof = oof[non_control_group_rows, :]

        all_oof[i, :, :] = oof
        blend_oof += oof * weight

        oof_loss = mean_logloss(oof, y_true)
        print(f"OOF Validation Loss of {script}: {oof_loss:.6f}\n")

    blend_oof_loss = mean_logloss(blend_oof, y_true)
    print(f"Blend OOF Validation Loss: {blend_oof_loss:.6f}\n")

elif search_mode and method == "optuna":
    print("[Optuna]")
    ## Search Best Blend Weights by Optuna ##
    model_oofs = []

    for i, (script, oof_filename, output_filename,
            weight) in enumerate(model_list):
        print(f"Loading OOF from {oof_filename} ......")
        oof = np.load(f"{dataset_folder}/{oof_filename}")

        if oof.shape[0] == 23814:
            oof = oof[non_control_group_rows, :]

        oof_loss = mean_logloss(
            oof, non_control_group_train_labels[train_classes].values)
        print(f"OOF Validation Loss of {script}: {oof_loss:.6f}\n")
        model_oofs.append(oof)

    def objective(trial):
        weights = []
        for i in range(len(model_list)):
            weights.append(trial.suggest_float(f"w{i}", 0, 1.0))

        blend = np.zeros(model_oofs[0].shape)
        for i in range(len(model_list)):
            blend += weights[i] * model_oofs[i]
        blend = np.clip(blend, 0, 1.0)

        loss = mean_logloss(
            blend, non_control_group_train_labels[train_classes].values)
        return loss

    pruner = optuna.pruners.MedianPruner(
        n_startup_trials=5,
        n_warmup_steps=0,
        interval_steps=1,
    )
    sampler = optuna.samplers.TPESampler(seed=rand_seed)
    study = optuna.create_study(direction="minimize",
                                pruner=pruner,
                                sampler=sampler,
                                study_name=study_name,
                                storage=f'sqlite:///{study_name}.db',
                                load_if_exists=True)

    study.optimize(objective,
                   n_trials=n_trials,
                   timeout=None,
                   gc_after_trial=True,
                   n_jobs=-1)

    trial = study.best_trial

    if run_submit_script:
        optimal_weights = []
        for i, (script, oof_filename, output_filename,
                _) in enumerate(model_list):
            optimal_weights.append(trial.params[f"w{i}"])
        submission = run_inference_scripts(submission, weights=optimal_weights)

    print("\n[Optuna]")
    print("Number of finished trials: {}".format(len(study.trials)))
    print("Best trial:")
    print("  Value: {}".format(trial.value))

    print("  Params: ")
    for key, value in trial.params.items():
        print("    {}: {}".format(key, value))

elif search_mode and method == "scipy":
    print("[Scipy SLSQP]")
    # Optimise Blending Weights with Bonus
    # https://www.kaggle.com/gogo827jz/optimise-blending-weights-with-bonus-0/notebook
    model_oofs = []
    y_true = non_control_group_train_labels[train_classes].values

    all_oof = np.zeros(
        (len(model_list), non_control_group_train_labels.shape[0], 206))
    print(all_oof.shape)
    for i, (script, oof_filename, output_filename,
            weight) in enumerate(model_list):
        print(f"Loading OOF from {oof_filename} ......")
        oof = np.load(f"{dataset_folder}/{oof_filename}")

        if oof.shape[0] == 23814:
            oof = oof[non_control_group_rows, :]

        all_oof[i, :, :] = oof

        oof_loss = mean_logloss(oof, y_true)
        print(f"OOF Validation Loss of {script}: {oof_loss:.6f}\n")
        model_oofs.append(oof)

    tol = 1e-10
    init_guess = [1 / all_oof.shape[0]] * all_oof.shape[0]
    bnds = [(0, 1) for _ in range(all_oof.shape[0])]
    cons = {
        'type': 'eq',
        'fun': lambda x: np.sum(x) - 1,
        'jac': lambda x: [1] * len(x)
    }

    print('Inital Blend OOF:', func_numpy_metric(init_guess))

    start_time = time.time()
    res_scipy = minimize(
        fun=func_numpy_metric,
        x0=init_guess,
        method='SLSQP',
        # jac=grad_func_jit,  # grad_func
        bounds=bnds,
        constraints=cons,
        tol=tol)
    print("\n[Scipy SLSQP]")
    print(
        f'[{str(datetime.timedelta(seconds = time.time() - start_time))[2:7]}] Optimised Blend OOF:',
        res_scipy.fun)
    print(f'Optimised Weights: {res_scipy.x}\n')

    if run_submit_script:
        submission = run_inference_scripts(submission, weights=res_scipy.x)

# Target-wise Weight Optimization #

elif search_mode and method == "scipy_per_target":
    print("[Scipy SLSQP]")
    # Optimise Blending Weights with Bonus
    # https://www.kaggle.com/gogo827jz/optimise-blending-weights-with-bonus-0/notebook
    model_oofs = []
    y_true = non_control_group_train_labels[train_classes].values

    all_oof = np.zeros(
        (len(model_list), non_control_group_train_labels.shape[0], 206))
    print(all_oof.shape)
    for i, (script, oof_filename, output_filename,
            weight) in enumerate(model_list):
        print(f"Loading OOF from {oof_filename} ......")
        oof = np.load(f"{dataset_folder}/{oof_filename}")

        if oof.shape[0] == 23814:
            oof = oof[non_control_group_rows, :]

        all_oof[i, :, :] = oof

        oof_loss = mean_logloss(oof, y_true)
        print(f"OOF Validation Loss of {script}: {oof_loss:.6f}\n")
        model_oofs.append(oof)

    print("\n[Scipy SLSQP Per Target]")
    optimized_target_weights = []
    for i, target in enumerate(train_classes):
        tol = 1e-10
        init_guess = [1 / all_oof.shape[0]] * all_oof.shape[0]
        bnds = [(0, 1) for _ in range(all_oof.shape[0])]
        cons = {
            'type': 'eq',
            'fun': lambda x: np.sum(x) - 1,
            'jac': lambda x: [1] * len(x)
        }

        def func_numpy_metric_targes(weights):
            oof_blend = np.tensordot(weights,
                                     all_oof[:, :, i],
                                     axes=((0), (0)))
            return log_loss_numpy(oof_blend, y_true[:, i])

        start_time = time.time()
        res_scipy = minimize(
            fun=func_numpy_metric_targes,
            x0=init_guess,
            method='SLSQP',
            # jac=grad_func_jit,  # grad_func
            bounds=bnds,
            constraints=cons,
            tol=tol)

        print(
            f'[{str(datetime.timedelta(seconds = time.time() - start_time))[2:7]}] ' + \
            f'Optimised Blend OOF for {target}:', res_scipy.fun)
        print(f'Optimised Weights for {target}: {res_scipy.x}\n')
        optimized_target_weights.append(res_scipy.x)

    blend_targets_oof = np.zeros(
        (non_control_group_train_labels.shape[0], 206))
    for i, (script, oof_filename, output_filename,
            weight) in enumerate(model_list):
        print(f"Loading OOF from {oof_filename} ......")
        oof = np.load(f"{dataset_folder}/{oof_filename}")

        if oof.shape[0] == 23814:
            oof = oof[non_control_group_rows, :]

        for j in range(206):
            blend_targets_oof[:,
                              j] += oof[:, j] * optimized_target_weights[j][i]

        oof_loss = mean_logloss(oof, y_true)
        print(f"OOF Validation Loss of {script}: {oof_loss:.6f}\n")

    blend_targets_oof_loss = mean_logloss(blend_targets_oof, y_true)
    print(
        f"Blend Target-Wise OOF Validation Loss: {blend_targets_oof_loss:.6f}\n"
    )

    # Save optimized weights per target
    save_pickle(optimized_target_weights, model_path,
                f"{study_name}_{blend_targets_oof_loss}")

    if run_submit_script:
        # For 206 target weights
        submission = run_inference_scripts(
            submission, target_weights=optimized_target_weights)
total_elapsed = time.time() - total_start
print(f"Total time spent: {total_elapsed/60:.2f} minutes.")
# [V7 - Improved 2heads, Removed SimpleNN]

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_2stageNN_ns_oldcv_10folds.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/2stageNN_with_ns_oldcv_10folds.py: 0.015461

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_improving-mark-s-2-heads-model_0.015660083675738144.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/improving-mark-s-2-heads-model-infer.py: 0.015660

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_efficientnet_lightning_v7_b3_0.01850.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_efficientnet_lightning_v7_b3_infer.py: 0.016016

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_ResNeSt_v1_resnest50_0.014619621213185928.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_resnest_lightning_v1_infer.py: 0.015819

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_ResNeSt_v2_resnest50_0.01854.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_resnest_lightning_v2_infer.py: 0.015756

# [Scipy SLSQP]
# [00:32] Optimised Blend OOF: 0.015107004167705938
# Optimised Weights: [0.36704255 0.0808577  0.17120266 0.10876667 0.27213043]
# [V7 - Improved 2heads, Mark's TabNet V2 Seeds]
# [Optuna]
# Number of finished trials: 3000
# Best trial:
#   Value: 0.015109516210552868
#   Params: 
#     w0: 0.345350187040706
#     w1: 0.018804585817609087
#     w2: 0.09890095339659595
#     w3: 0.15486138107425962
#     w4: 0.1302653777195202
#     w5: 0.24674004919729609
#     w6: 0.00020099415280697705

# [Scipy SLSQP]
# [00:32] Optimised Blend OOF: 0.015107023232510362
# Optimised Weights: [0.36394211 0.         0.08646557 0.17174434 0.1059386  0.27190938
#  0.        ]
# [V7 - without TabNet, Mark's 2heads, 10-folds 2StageNN and SimpleNN, ResNeSt V1]

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_2stageNN_ns_oldcv_10folds.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/2stageNN_with_ns_oldcv_10folds.py: 0.015461

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_NN_oldcv_10fold.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/script_simpleNN_oldcv_10folds.py: 0.015741

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_improving-mark-s-2-heads-model_0.015660083675738144.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/improving-mark-s-2-heads-model-infer.py: 0.015660

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_efficientnet_lightning_v7_b3_0.01850.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_efficientnet_lightning_v7_b3_infer.py: 0.016016

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_ResNeSt_v2_resnest50_0.01854.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_resnest_lightning_v2_infer.py: 0.015756

# [Optuna]
# Number of finished trials: 5000
# Best trial:
#   Value: 0.015118051559985067
#   Params: 
#     w0: 0.3693123925185777
#     w1: 0.0004358344602283515
#     w2: 0.11986363820431883
#     w3: 0.18219399304034917
#     w4: 0.32927717286142916

# [Scipy SLSQP]
# [00:24] Optimised Blend OOF: 0.015117984713726062
# Optimised Weights: [3.79183409e-01 8.13151629e-19 1.08023618e-01 1.86335277e-01
#  3.26457696e-01]
# [V7 - without TabNet, Mark's 2heads, 10-folds 2StageNN and SimpleNN]
# [Optuna]
# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_2stageNN_ns_oldcv_10folds.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/2stageNN_with_ns_oldcv_10folds.py: 0.015461

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_NN_oldcv_10fold.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/script_simpleNN_oldcv.py: 0.015741

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_improving-mark-s-2-heads-model_0.015660083675738144.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/improving-mark-s-2-heads-model-infer.py: 0.015660

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_efficientnet_lightning_v7_b3_0.01850.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_efficientnet_lightning_v7_b3_infer.py: 0.016016

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_ResNeSt_v1_resnest50_0.014619621213185928.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_resnest_lightning_v1_infer.py: 0.015819

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_ResNeSt_v2_resnest50_0.01854.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_resnest_lightning_v2_infer.py: 0.015756

# [Optuna]
# Number of finished trials: 10000
# Best trial:
#   Value: 0.015107277135046551
#   Params: 
#     w0: 0.36295047115403034
#     w1: 0.00019665601658366993
#     w2: 0.10383226438101152
#     w3: 0.16907012564901389
#     w4: 0.10211240990528128
#     w5: 0.26308884808653726

# [Scipy SLSQP]
# [00:48] Optimised Blend OOF: 0.015107004381942337
# Optimised Weights: [0.36727479 0.         0.08130163 0.17091322 0.10863643 0.27187394]
# [V7 - without TabNet, Mark's 2heads]
# [Optuna]
# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_2stageNN_ns_oldcv.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/2stageNN_with_ns_oldcv.py: 0.015606

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_script_simpleNN_oldcv.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/script_simpleNN_oldcv.py: 0.015846

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_improving-mark-s-2-heads-model_0.015660083675738144.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/improving-mark-s-2-heads-model-infer.py: 0.015660

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_efficientnet_lightning_v7_b3_0.01850.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_efficientnet_lightning_v7_b3_infer.py: 0.016016

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_ResNeSt_v1_resnest50_0.014619621213185928.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_resnest_lightning_v1_infer.py: 0.015819

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_ResNeSt_v2_resnest50_0.01854.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_resnest_lightning_v2_infer.py: 0.015756

# [Optuna]
# Number of finished trials: 10000
# Best trial:
#   Value: 0.015146055466812214
#   Params: 
#     w0: 0.2590485239176548
#     w1: 0.033637086725830254
#     w2: 0.12194872432858535
#     w3: 0.19075874431151257
#     w4: 0.11084230101870868
#     w5: 0.28295491196215444

# [Scipy SLSQP]
# [00:30] Optimised Blend OOF: 0.01514578929579469
# Optimised Weights: [0.25256016 0.03296526 0.13166163 0.18166774 0.11849445 0.28265076]
# [V7 - without TabNet]
# [Optuna]
# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_2stageNN_ns_oldcv.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/2stageNN_with_ns_oldcv.py: 0.015606

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_script_simpleNN_oldcv.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/script_simpleNN_oldcv.py: 0.015846

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_fork-of-2heads-looper-super-puper-markpeng.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/fork-of-2heads-looper-super-puper-markpeng-infer.py: 0.015887

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_improving-mark-s-2-heads-model_0.015660083675738144.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/improving-mark-s-2-heads-model-infer.py: 0.015660

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_efficientnet_lightning_v7_b3_0.01850.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_efficientnet_lightning_v7_b3_infer.py: 0.016016

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_ResNeSt_v1_resnest50_0.014619621213185928.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_resnest_lightning_v1_infer.py: 0.015819

# Loading OOF from ../../Github/kaggle_moa_team/oof/oof_deepinsight_ResNeSt_v2_resnest50_0.01854.npy ......
# OOF Validation Loss of ../../Github/kaggle_moa_team/scripts/deepinsight_resnest_lightning_v2_infer.py: 0.015756

# Number of finished trials: 10000
# Best trial:
#   Value: 0.015146827058312776
#   Params: 
#     w0: 0.2507759947969869
#     w1: 0.03339376719044784
#     w2: 8.528382280836113e-05
#     w3: 0.11625864567289637
#     w4: 0.18770116032490702
#     w5: 0.09530145379465743
#     w6: 0.3164264877487286

# [Scipy SLSQP]
# [00:45] Optimised Blend OOF: 0.015145788791365362
# Optimised Weights: [2.52672054e-01 3.32734786e-02 1.05430191e-17 1.30957268e-01
#  1.82060938e-01 1.18387855e-01 2.82648406e-01]
# [V6 - without TabNet, 2heads]
# Total time spent: 0.68 minutes.
# Blend Target-Wise OOF Validation Loss: 0.015044
# [V6 - without TabNet, 2heads]
# [Optuna]
# Number of finished trials: 3000
# Best trial:
#   Value: 0.015171999561900233
#   Params: 
#     w0: 0.323528084383917
#     w1: 0.08786476491118465
#     w2: 0.21849845883367852
#     w3: 0.3704230222796271

# [Scipy SLSQP]
# [00:22] Optimised Blend OOF: 0.015172004593585666
# Optimised Weights: [0.32020133 0.09043987 0.22122948 0.36812932]
# [V6 - without TabNet]
# [Optuna]
# Number of finished trials: 3000
# Best trial:
#   Value: 0.015172424601530761
#   Params: 
#     w0: 0.3138176484100186
#     w1: 0.07850519440561339
#     w2: 0.0007183363099561991
#     w3: 0.23849563017967007
#     w4: 0.3694870328388392

# [Scipy SLSQP]
# [00:21] Optimised Blend OOF: 0.015172004898867827
# Optimised Weights: [0.32045559 0.09026525 0.         0.22069638 0.36858278]
# [V6]
# [Optuna]
# Number of finished trials: 5000
# Best trial:
#   Value: 0.015173437622007157
#   Params: 
#     w0: 0.30923325055652684
#     w1: 0.09831493504786226
#     w2: 0.018966959973949222
#     w3: 0.19863369862866234
#     w4: 0.0013224625996093413
#     w5: 0.3728865483320761

# [Scipy SLSQP]
# [00:36] Optimised Blend OOF: 0.015172005464591968
# Optimised Weights: [3.20472642e-01 9.01191588e-02 1.78893358e-18 2.20448482e-01
#  3.27971157e-18 3.68959717e-01]
# [V5]
# Number of finished trials: 3000
# Best trial:
#   Value: 0.015344701181290615
#   Params: 
#     w0: 0.5141433844379889
#     w1: 0.11747776562133813
#     w2: 0.3668324643717302

# [00:14] Optimised Blend OOF: 0.015344695215068541
# Optimised Weights: [0.51922623 0.11292509 0.36784869]
# [V4]
# [Optuna]
# Number of finished trials: 3000
# Best trial:
#   Value: 0.015331901615194453
#   Params: 
#     w0: 0.4505928450756189
#     w1: 0.13010257032841785
#     w2: 0.06308933354044946
#     w3: 0.35639153615958885
#
# [Scipy]
# [00:23] Optimised Blend OOF: 0.015331777381591449
# Optimised Weights: [0.44090106 0.14508641 0.05945655 0.35455598]
# [V3]
# improving-mark-s-2-heads-model-infer
# Number of finished trials: 3000
# Best trial:
#   Value: 0.01515466145873492
#   Params: 
#     w0: 0.0002980690037490555
#     w1: 0.29771381784976886
#     w2: 0.1569191862042946
#     w3: 0.18156875605872544
#     w4: 0.36371774630338105
# [V3]
# fork-of-2heads-looper-super-puper-markpeng-infer
# Number of finished trials: 3000
# Best trial:
#   Value: 0.015170138066049686
#   Params: 
#     w0: 0.00019903389488299251
#     w1: 0.3853752127955825
#     w2: 0.015968332256452233
#     w3: 0.22945916769823432
#     w4: 0.3711290150522236
if search_mode and method == "scipy_per_target":
    # OOF scores per target
    target_oof_losses = []
    for i, target in enumerate(train_classes):
        print(target)
        # print(y_true[:, i])

        oof_loss = mean_logloss(blend_targets_oof[:, i], y_true[:, i])
        target_oof_losses.append(oof_loss)
        print(f"Blend OOF Validation Loss of {target}: {oof_loss:.6f}\n")

    target_loss_df = pd.DataFrame(
        data={
            "target": train_classes,
            "oof_logloss": target_oof_losses
        },
        columns=["target", "oof_logloss"
                 ]).sort_values(by="oof_logloss",
                                ascending=False).reset_index(drop=True)
    print(target_loss_df)
if run_submit_script:
    print(submission.shape)
    print(submission)
    submission.to_csv('submission.csv', index=False)<jupyter_output><empty_output><jupyter_text>EOF<jupyter_code>if kernel_mode:
    !rm ./*.py
    !ls -la<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13150320058426773, 'fraction_numerical': 0.12690638828027667, 'mean_word_length': 3.0660262008733623, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '328178', 'license': 'Apache-2.0', 'n_tokens_mistral': 11115, 'n_tokens_neox': 8946, 'n_words': 1453, 'path': 'blends/blend_search_optuna_v7.ipynb', 'repo_name': 'mrblasco/kaggle_moa_winner_hungry_for_gold', 'size': '23277'}