<jupyter_start><jupyter_text>**Section:** Unbalance different size<jupyter_code>import os
import itertools as it
import warnings
import time
import pickle

import numpy as np

import pandas as pd

%matplotlib inline
from matplotlib import pyplot as plt
from matplotlib import cm
import seaborn as sns

import joblib

import pathlib

from sklearn.ensemble import RandomForestClassifier
from sklearn.exceptions import DataConversionWarning
from sklearn.model_selection import (
    KFold, StratifiedKFold, train_test_split)

from sklearn import metrics

import tqdm

from libs.container import Container
from libs.nearest import nearest
from libs.experiment import WithAnotherExperiment, roc, metrics
from libs.precstar import  prec_star

warnings.simplefilter("ignore", category=DataConversionWarning)
PATH = pathlib.Path(os.path.abspath(os.path.dirname("")))

DATA_PATH = PATH / "_data"

COLUMNS_NO_FEATURES = ['id', 'tile', 'cnt', 'ra_k', 'dec_k', 'vs_type', 'vs_catalog', 'cls'] 

tiles = ["b234", "b360", "b278", "b261"]
%%time

path = "_data/full_scaled.pkl.bz2"


train_datas = {}

sample = pd.read_pickle(path)

# the features
X_columns = [c for c in sample.columns if c not in COLUMNS_NO_FEATURES]
y_column = "cls"

sample[X_columns] =  sample[X_columns].astype(np.float32)

data = Container({k: v for k, v in sample.groupby("tile") if k in tiles})

del sample

REAL_SIZES = {'b234': 293013, 'b261': 555693, 'b278': 742153, 'b360': 939110}<jupyter_output>CPU times: user 4min 35s, sys: 6min 7s, total: 10min 42s
Wall time: 1min 54s<jupyter_text>Run the classifiers with the selected parameters<jupyter_code>RF_PARAMS = joblib.load("_cache/best_params.pkl.bz2")["rf"]
if "n_jobs" in RF_PARAMS:
    del RF_PARAMS["n_jobs"]
# RF_PARAMS["n_jobs"]  = -1
RF_PARAMS
def get_combs(data, X_columns, nfolds):
    combs = []
    for idx, elem in enumerate(data.items()):
        tile, df = elem
        combs.append({
            "idx": idx,
            "tile": tile,
            "test_sample": df,
            "X_columns": X_columns,
            "y_column": "cls",
            "nfolds": nfolds})
    return combs
    

def execute_clf(idx, tile, test_sample,  X_columns, y_column, nfolds):
    
    skf = StratifiedKFold(n_splits=nfolds)
    
    X = test_sample[X_columns].values
    y = test_sample[y_column].values
    
    probabilities = None
    predictions = np.array([])
    y_tests = np.array([])
    
    for train, test in skf.split(X, y):
        x_train = X[train]
        y_train = y[train]
        x_test = X[test]
        y_test = y[test]
        
        clf = RandomForestClassifier(**RF_PARAMS)
        clf.fit(x_train, y_train)
        
        current_pred = clf.predict(x_test)
        current_prob = clf.predict_proba(x_test)
        
        y_tests = np.hstack([y_tests, y_test])
        predictions = np.hstack([predictions, current_pred])
        probabilities = (
                current_prob if probabilities is None else
                np.vstack([probabilities, current_prob ]))
            
    fpr, tpr, thresholds = metrics.roc_curve(
        y_tests, 1.-probabilities[:,0], pos_label=1)
    prec_rec_curve = metrics.precision_recall_curve(
        y_tests, 1.- probabilities[:,0], pos_label=1)
    
    roc_auc = metrics.auc(fpr, tpr)
    
    result = Container({
        "idx": idx,
        "clf_name": "RF",
        "train_name": tile,
        "test_name": f"KFolds ({nfolds})",
        'fpr': fpr,
        'tpr': tpr,
        'thresh': thresholds,
        'roc_auc': roc_auc,
        'prec_rec_curve': prec_rec_curve,
        'real_cls': y_tests,
        'predictions': predictions,
        'probabilities': probabilities,
        'confusion_matrix': metrics.confusion_matrix(y_tests, predictions)})    
    return result

def train_and_run(data, X_columns):
    combs = get_combs(data, X_columns, 10)
    print("Combinaciones: {}".format(len(combs)))    
    with joblib.Parallel(n_jobs=-1) as jobs:
        results = jobs(
            joblib.delayed(execute_clf)(**p) for p in tqdm.tqdm(combs))
    return results
if os.path.exists("_cache/kfolds_results.pkl.bz2"):
    kfolds_results = joblib.load("_cache/kfolds_results.pkl.bz2")
else:
    kfolds_results = train_and_run(data, X_columns)
    joblib.dump(kfolds_results, "_cache/kfolds_results.pkl.bz2", compress=3)
def get_fix_threshold(results, recall):
    fix_thresholds = {}
    for r in results:
        train_name = r["train_name"]
        
        if train_name not in fix_thresholds:
            fix_thresholds[train_name] = {}
            
        precs, recalls, probs = r.prec_rec_curve
        
        idx = nearest(recalls, recall)
        
        fix_thresholds[train_name] = probs[idx]
    return fix_thresholds
    
thresholds = get_fix_threshold(kfolds_results, 0.9)
thresholds
%%time
def make_full_classiffiers(data):
    clfs = {}
    for tile, tdata in data.items():
        print(tile)
        X = tdata[X_columns].values
        y = tdata[y_column].values
        clf = RandomForestClassifier(n_jobs=10, **RF_PARAMS)
        clf.fit(X, y)
        clfs[tile] = clf
    return clfs

if os.path.exists("_cache/kfolds_full_clfs.pkl.bz2"):
    full_clfs = joblib.load("_cache/kfolds_full_clfs.pkl.bz2")
else:
    full_clfs = make_full_classiffiers(data)
    joblib.dump(full_clfs, "_cache/kfolds_full_clfs.pkl.bz2", compress=3)
%%time
def get_combs_full(clfs, data, X_columns, kfold_thresholds):
    combs = []
    for train_name, clf in clfs.items():
        for test_name in clfs.keys():
            if train_name != test_name:
                test_sample = data[test_name]
                comb = Container({
                    "idx": len(combs), 
                    "train_name": train_name, "clf": clf, 
                    "kfold_thrs": kfold_thresholds[train_name],
                    "test_name": test_name, "test_sample": test_sample,
                    "X_columns": X_columns, "y_column": y_column})
                combs.append(comb)
    return combs

def execute_full_clf(idx, train_name, clf, test_name, test_sample, X_columns, y_column, kfold_thrs):
    
    X_test = test_sample[X_columns].values
    y_test = test_sample[y_column].values
    
    predictions = clf.predict(X_test)
    probabilities = clf.predict_proba(X_test)
    
    fpr, tpr, thresholds = metrics.roc_curve(
        y_test, 1.-probabilities[:,0], pos_label=1)

    prec_rec_curve = metrics.precision_recall_curve(
        y_test, 1.- probabilities[:,0], pos_label=1)

    roc_auc = metrics.auc(fpr, tpr)
    
    result = Container({
        "idx": idx,
        "train_name": train_name,
        "test_name": test_name,
        'fpr': fpr,
        'tpr': tpr,
        'thresh': thresholds,
        'roc_auc': roc_auc,
        'prec_rec_curve': prec_rec_curve,
        'real_cls': y_test,
        'predictions': predictions,
        'probabilities': probabilities,
        "kfold_thrs": kfold_thrs,
        'confusion_matrix': metrics.confusion_matrix(y_test, predictions)})    
    return result


def run_full(full_clfs, data, X_columns, kfold_thresholds):
    combs = get_combs_full(full_clfs, data, X_columns, kfold_thresholds)
    print("Combinaciones: {}".format(len(combs)))
    print("Launching classifiers for {} features...".format(len(X_columns)))
    with joblib.Parallel(n_jobs=-1) as jobs:
        results = jobs(
            joblib.delayed(execute_full_clf)(**comb) for comb in tqdm.tqdm(combs))
    return results

results = run_full(full_clfs, data, X_columns, thresholds)

def as_df(data):
    rows, tiles = [], sorted(list(data.keys()))
    for rname in tiles:
        row = data[rname].copy()
        row.update({"Tile": rname}) 
        rows.append(row)
    df = pd.DataFrame(rows)
    df = df.set_index("Tile")
    return df.T

def heatmap(ax, results, fp):
    cmap = sns.cm.rocket
    
    fix_recall, fix_precs, fix_thresholds = {}, {}, {}
    for r in results:
        train_name, test_name = r["train_name"], r["test_name"]
        
        if train_name not in fix_recall:
            fix_recall[train_name] = {}
            fix_precs[train_name] = {}
            fix_thresholds[train_name] = {}
            
        precs, recalls, probs = r.prec_rec_curve
        
        idx = nearest(recalls, fp)
        
        fix_recall[train_name][test_name] = recalls[idx]
        fix_precs[train_name][test_name] = precs[idx]
        fix_thresholds[train_name][test_name] = probs[idx]
#     return fix_precs
        
    fix_precs = as_df(fix_precs)
    fix_recall = as_df(fix_recall)
    
    sns.heatmap(fix_precs, annot=True, fmt='.3f',  linewidths=.5, ax=ax[0], cmap=cmap, center=.5, vmin=0., vmax=1.)
    ax[0].set_xlabel("Test")
    ax[0].set_title(u"Precision".format(fp))
    
    sns.heatmap(fix_recall, annot=True, fmt='.3f', linewidths=.5, ax=ax[1], cmap=cmap, center=.5, vmin=0., vmax=1.)
    ax[1].set_xlabel("Test")
    ax[1].set_title(u"Recall".format(fp))


fig, axs = plt.subplots(1, 2, figsize=(10, 2))
heatmap(axs, kfolds_results, 0.90)
axs[0].set_ylabel("K-Folds")

axs[0].set_yticklabels([""])
axs[1].set_yticklabels([""])
    
fig.tight_layout()
# fig.savefig(plot_path / "section_unbalanced_diffsize_prec_rec_heatmap.pdf")
def plot_prec_roc_curve(ax, train_name, results):
    for r in results:
        if r.train_name == train_name:
            ax.plot(r.prec_rec_curve[1], r.prec_rec_curve[0], label=train_name)
            break
    ax.set_title("K-Fold")
    ax.set_xlabel("Recall")
    ax.set_ylabel("Precision")
    ax.legend(loc='upper right')

fig, ax = plt.subplots(1, 1, figsize=(9, 5))
for t in data.keys():
    plot_prec_roc_curve(ax, train_name=t, results=kfolds_results)

fig.tight_layout()
def as_df(data):
    rows, tiles = [], sorted(list(data.keys()))
    for rname in tiles:
        row = data[rname].copy()
        row.update({"Train": rname}) 
        rows.append(row)
    df = pd.DataFrame(rows)
    df = df.set_index("Train")[tiles]
    return df

def heatmap(ax, results):
    cmap = sns.cm.rocket
    
    fix_recall, fix_precs = {}, {}
    for r in results:
        train_name, test_name = r["train_name"], r["test_name"]
        kfold_thrs = r["kfold_thrs"]
        
        if train_name not in fix_recall:
            fix_recall[train_name] = {}
            fix_precs[train_name] = {}
            
        precs, recalls, probs = r.prec_rec_curve
        
        idx = nearest(probs, kfold_thrs)
        
        fix_recall[train_name][test_name] = recalls[idx]
        fix_precs[train_name][test_name] = precs[idx]
        
    fix_precs = as_df(fix_precs)
    fix_recall = as_df(fix_recall)
    
    sns.heatmap(fix_precs, annot=True, fmt='.3f',  linewidths=.5, ax=ax[0], cmap=cmap, center=.5, vmin=0., vmax=1.)
    ax[0].set_xlabel("Test")
    ax[0].set_title(u"Precision")
    
    sns.heatmap(fix_recall, annot=True, fmt='.3f', linewidths=.5, ax=ax[1], cmap=cmap, center=.5, vmin=0., vmax=1.)
    ax[1].set_xlabel("Test")
    ax[1].set_title(u"Recall")

fig, axes = plt.subplots(1, 2, figsize=(10, 3))


heatmap(axes, results)
# axs[0].set_ylabel(f"{rname}\n{axs[0].get_ylabel()}")
len(results)
colors.pop?
colors = dict(zip(tiles, cmap(np.linspace(0, 1, 4))))
labels = set()

def plot_prec_roc_curve(ax, train_name, test_name, all_results):
    color = colors[test_name]
    for r in results:
        if r.test_name == test_name and r.train_name == train_name:
            label = test_name
            if label not in labels:
                labels.add(label)
            else:
                label = "__nolegend__"
            ax.plot(r.prec_rec_curve[1], r.prec_rec_curve[0], label=label, color=color)
            break
    ax.set_title("Train {}".format(train_name))
    ax.set_xlabel("Recall")
    ax.set_ylabel("Precision")

fig, axes = plt.subplots(2, 2, figsize=(18-6, 10))
axes = it.chain(*axes)
for train_name in sorted(tiles):
    ax = next(axes)
    for test_name in tiles:
        if train_name == sorted(test_name):
            continue
        plot_prec_roc_curve(ax, train_name, test_name, results)

fig.legend(loc=7, bbox_to_anchor=(1.12, 0.5))
fig.tight_layout()
labels
colors
import datetime
datetime.datetime.now()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1291889666529436, 'fraction_numerical': 0.013750514615067929, 'mean_word_length': 2.957640925382861, 'pattern_counts': {'":': 24, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '156936', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4606, 'n_tokens_neox': 4395, 'n_words': 853, 'path': '12_kfolds.ipynb', 'repo_name': 'leliel12/How-far-can-we-can-go', 'size': '12145'}