<jupyter_start><jupyter_text>Setup directories<jupyter_code># data directories
DATA_DIR = os.path.join('..', 'data')
SUBMISSION_DIR = os.path.join('..', 'submission')
BAYESIAN_OPTS_DIR = 'bayesian-opts-res'
BAYESIAN_OPTS_TEST_DIR = os.path.join(BAYESIAN_OPTS_DIR, 'bayesian-opt-test-preds')

# Setup directories
if not os.path.isdir(BAYESIAN_OPTS_DIR):
    logging.info('Creating {} directory...'.format(BAYESIAN_OPTS_DIR))
    os.mkdir(BAYESIAN_OPTS_DIR)

if not os.path.isdir(BAYESIAN_OPTS_TEST_DIR):
    logging.info('Creating {} directory...'.format(BAYESIAN_OPTS_TEST_DIR))
    os.mkdir(BAYESIAN_OPTS_TEST_DIR)

if not os.path.isdir(SUBMISSION_DIR):
    logging.info('Creating {} directory...'.format(SUBMISSION_DIR))
    os.mkdir(SUBMISSION_DIR)<jupyter_output><empty_output><jupyter_text>Define and initialize variables<jupyter_code>bayes_opt_verbose = True  # Flag for bayes-opt logging
log_verbose = True  # Flag for miscellaneous log logging
init_points = 2  # Number of random initialization rounds (30 used in winning submission)
opt_iter = 2  # Number of bayesian optimization rounds (70 used in winning submission)

num_gen_params =  2  # Number of top meta predictions to use for blending (20 used in winning submission)

round_num = None  # Global variable to keep track of the bayesian optimization round
is_training = None  # Global variable to indicate training or testing periods

#session = datetime.now()
opt_val_seeds = get_opt_val_seeds(init_points * opt_iter)  # Generation of seeds for each bayesian optimization round

model_set = ['lr','nn','xgb', 'rf', 'lgb']

opt_res = dict()

preds_dict = dict()<jupyter_output><empty_output><jupyter_text>Execute training and meta prediction inference<jupyter_code>with timing('Performing training and meta predictions'):
    x_model_train_preds = pd.DataFrame()
    x_model_test_preds = pd.DataFrame()
    with timing('Start bayesian optimization and generation of optimal sub-models'):
        for ix, model_type in enumerate(model_set):
            with timing('{}. Training bayesian opts for {} model.'.format(ix + 1, model_type)):
                modelBO = bayesian_optimize_model(model_type=model_type, tunable_params=None, num_iter=opt_iter, init_points=init_points)
            opt_res[model_type] = modelBO.res
with timing('Performing training and meta predictions'):
    x_model_train_preds = pd.DataFrame()
    x_model_test_preds = pd.DataFrame()
    with timing('Start bayesian optimization and generation of optimal sub-models'):
        for ix, model_type in enumerate(model_set):
            with timing('{}. Training bayesian opts for {} model.'.format(ix + 1, model_type)):
                modelBO = bayesian_optimize_model(model_type=model_type, tunable_params=None, num_iter=opt_iter, init_points=init_points)
            opt_res[model_type] = modelBO.res
    fname = 'bayesian-opts-res/bayesian_opt_res_results.pkl'
    vprint(fname)

    with open(fname, 'wb') as fl:
        cPickle.dump(opt_res, fl)
        
    with timing('Generate inference using the top {} models based on bayesian OOF score'.format(num_gen_params)):
        for ix, model_type in enumerate(model_set):
            with timing('{}. Generating meta predictions for {} model.'.format(ix + 1, model_type)):
                model_test_preds, model_train_preds = predict_from_opt(
                    model_type, num_params=num_gen_params,
                    use_latest=True, default_dir=BAYESIAN_OPTS_DIR
                )


                model_test_preds.to_csv(
                    os.path.join(BAYESIAN_OPTS_TEST_DIR, 'model_test_preds.csv'),
                    'model_test_preds_{}'.format(model_type)
                )

                model_train_preds.to_csv(
                    os.path.join(BAYESIAN_OPTS_TEST_DIR, 'model_train_preds.csv'),
                    'model_train_preds{}'.format(model_type)
                )

                # Collect meta predictions
                x_model_train_preds = pd.concat([x_model_train_preds, model_train_preds], axis=1)
                x_model_test_preds = pd.concat([x_model_test_preds, model_test_preds], axis=1)
    
    preds_dict['train'] = x_model_train_preds
    preds_dict['test'] = x_model_test_preds
    preds_dict['y_train'] = train.loc[x_model_train_preds.index].SeriousDlqin2yrs.values<jupyter_output><empty_output><jupyter_text>Perform OOF optimized blending<jupyter_code>median_weight_optimized_preds_sub = []
mean_weight_optimized_preds_sub = []


with timing('Optimizing meta prediction weights'):
    x_test_median_preds, x_test_mean_preds, x_optimized_weights_preds, x_coeffs = get_optimized_weighted_preds_for(preds_dict)

median_weight_optimized_preds_sub.append(make_sub(x_test_median_preds, x_test_median_preds))
mean_weight_optimized_preds_sub.append(make_sub(x_test_mean_preds, x_test_mean_preds))

median_weight_optimized_preds_sub = pd.concat(median_weight_optimized_preds_sub)
mean_weight_optimized_preds_sub = pd.concat(mean_weight_optimized_preds_sub)


date_now = datetime.now()
median_sub_fname = os.path.join(SUBMISSION_DIR, 'submission-test-median-agg-models.csv')
mean_sub_fname = os.path.join(SUBMISSION_DIR, 'submission-test-mean-agg-models.csv')

logging.info(median_sub_fname)
logging.info(mean_sub_fname)

median_weight_optimized_preds_sub.to_csv(median_sub_fname, index=False)
mean_weight_optimized_preds_sub.to_csv(mean_sub_fname, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09563294637921503, 'fraction_numerical': 0.002763957987838585, 'mean_word_length': 4.417165668662674, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '265158', 'license': 'MIT', 'n_tokens_mistral': 1892, 'n_tokens_neox': 1797, 'n_words': 321, 'path': 'src/.ipynb_checkpoints/Credit_Full Bayesian Model Training and Predictions-checkpoint.ipynb', 'repo_name': 'rafmacalaba/give-me-some-credit-kaggle', 'size': '5427'}