<jupyter_start><jupyter_text>HypothesisOur final ensemble (and all L2 models) is not predicting anything under 20. Creating a classifier able to identify samples with a Pawpularity under 20 could improve the final ensemble.<jupyter_code>from ml.vision.data import ImageDataModule
from ml.learner import ImageClassifier
from ml.params import load_cfg
import pandas as pd

df = pd.read_csv("../data/train_folds.csv")
df.head()
train_image_fpaths = [f"../data/train/{i}.jpg" for i in df[df.kfold!=0]["Id"]]
train_targets = [[1] if t <= 20 else [0] for t in df[df.kfold!=0].Pawpularity.tolist()]
val_image_fpaths = [f"../data/train/{i}.jpg" for i in df[df.kfold==0]["Id"]]
val_targets = [[1] if t <= 20 else [0] for t in df[df.kfold==0].Pawpularity.tolist()]
train_image_fpaths[:5]
train_targets[:5]
cfg = load_cfg("../params.yaml", cfg_name="train_two")
cfg.metric = 'auc'
cfg.metric_mode = 'max'
cfg
from timm.data import transforms_factory

train_aug = transforms_factory.create_transform(
    input_size=cfg.sz,
    is_training=True,
    auto_augment=f"rand-n{cfg.n_tfms}-m{cfg.magn}",
)
val_aug = transforms_factory.create_transform(
    input_size=cfg.sz,
    is_training=False,
)
dm = ImageDataModule(
    task="classification",
    batch_size=cfg.bs,
    # train
    train_image_paths=train_image_fpaths,
    train_targets=train_targets,
    train_augmentations=train_aug,
    # valid
    val_image_paths=val_image_fpaths,
    val_targets=val_targets,
    val_augmentations=val_aug,
    # test
    test_image_paths=val_image_fpaths,
    test_augmentations=val_aug,
)
model = ImageClassifier(
    in_channels=3,
    num_classes=1,
    pretrained=cfg.pretrained,
    cfg=cfg,
)
import pytorch_lightning as pl 

trainer = pl.Trainer(
    gpus=1,
    precision=cfg.precision,
    auto_lr_find=cfg.auto_lr,
    accumulate_grad_batches=cfg.accumulate_grad_batches,
    auto_scale_batch_size=cfg.auto_batch_size,
    max_epochs=cfg.epochs,
#     logger=logger,
#     callbacks=[checkpoint_callback, lr_callback],
    # limit_train_batches=1,
    # limit_val_batches=1,
)
trainer.fit(model, dm)
preds = trainer.predict(model, dm.test_dataloader(), ckpt_path="best")
preds_list = [p[0] for b in preds for p in b]
import numpy as np

preds = np.array(preds_list)
preds.min(), preds.max()
import matplotlib.pyplot as plt

plt.hist(preds, bins=50);
from sklearn.metrics import precision_score

precision_score(val_targets, preds> 0.50)
np.sum(val_targets) / len(val_targets)
from sklearn import metrics

fpr, tpr, thresholds = metrics.roc_curve(val_targets, preds)
np.argmax(tpr - fpr)
from sklearn.metrics import roc_curve

def Find_Optimal_Cutoff(target, predicted):
    """ Find the optimal probability cutoff point for a classification model related to event rate
    Parameters
    ----------
    target : Matrix with dependent or target data, where rows are observations

    predicted : Matrix with predicted data, where rows are observations

    Returns
    -------     
    list type, with optimal cutoff value
        
    """
    fpr, tpr, threshold = roc_curve(target, predicted)
    i = np.arange(len(tpr)) 
    roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})
    roc_t = roc.iloc[(roc.tf-0).abs().argsort()[:1]]

    return list(roc_t['threshold']) 


threshold = Find_Optimal_Cutoff(val_targets, preds)
threshold
np.sum(preds > 0.16)
preds<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12023289665211062, 'fraction_numerical': 0.010480349344978166, 'mean_word_length': 4.453968253968254, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '364516', 'license': 'MIT', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1217, 'n_words': 267, 'path': 'nbs/under_20.ipynb', 'repo_name': 'IamGianluca/petfinder-pawpularity-score', 'size': '3435'}