<jupyter_start><jupyter_text>Evaluate the performance of a model on the test set<jupyter_code>%load_ext autoreload
%autoreload 2
from pathlib import Path

import holoviews as hv
import hvplot.pandas
import janitor
import numpy as np
import pandas as pd
import torch

from pytorch_hcs.datasets import BBBC021DataModule
from pytorch_hcs.models import ResNet18, ResNet101, ResNet18Embeddings

from sklearn.metrics import confusion_matrix
from tqdm.notebook import tqdm

hv.extension('bokeh')
data_path = Path("data")
data_path<jupyter_output><empty_output><jupyter_text>Choose GPU or CPU processing<jupyter_code>DEVICE = "cuda"
# DEVICE = 'cpu'<jupyter_output><empty_output><jupyter_text>Initialize W&B runThis is only so we can load model checkpoints from W&B artifacts.Future TODO would be to store the results of the evaluation run in W&B.<jupyter_code>import wandb

wandb.login()
run = wandb.init(project='pytorch-hcs', name='evaluation')<jupyter_output><empty_output><jupyter_text>Specify model to loadModel checkpoint artifacts from the training will be accessible under `'model-{version}'`,where `version` is by default set to the class name of the PyTorch-Lightning module.You can find your model artifact names as a tab in the left pane under the W&B run.<jupyter_code>user_project = 'zbarry/pytorch-hcs'

model_id, model_cls = "resnet18:latest", ResNet18
# model_id, model_cls = "resnet101:latest", ResNet101
# model_id, model_cls = "resnet18-embeddings:latest", ResNet18Embeddings<jupyter_output><empty_output><jupyter_text>Load modelDownload model .ckpt file from W&B. Note that a model `.ckpt` file can be loaded directly through `.load_from_checkpoint` in the `data/weights` directoryrather than downloaded from W&B, if desired.<jupyter_code>artifact = run.use_artifact(
    f"{user_project}/{model_id}", type="model"
)

artifact_dir = artifact.download()

ckpt_path = f"{artifact_dir}/model.ckpt"

model = model_cls.load_from_checkpoint(str(ckpt_path)).eval().to(DEVICE)<jupyter_output><empty_output><jupyter_text>Set up `LightningDataModule`<jupyter_code>dm = BBBC021DataModule(
    num_workers=8 if DEVICE != "cpu" else 0,  # h5py pickling error otherwise...
    tv_batch_size=16,
    t_batch_size=8,
)

dm.setup()<jupyter_output><empty_output><jupyter_text>Iterate through dataset, extracting predicted class labels from modelTraining and validation datasets also available for exploration.<jupyter_code># dataset = dm.train_dataset
# dataloader = dm.train_dataloader()

# dataset = dm.val_dataset
# dataloader = dm.val_dataloader()

dataset = dm.test_dataset
dataloader = dm.test_dataloader()
true_labels = []
predicted_labels = []
image_idcs = []

with torch.no_grad():
    for image_batch, true_labels_batch, metadata_batch in tqdm(dataloader):
        image_idcs.extend(metadata_batch.image_idx.numpy())
        outputs = model(image_batch.to(DEVICE))

        labels = torch.argmax(outputs, 1).cpu()

        predicted_labels.extend(labels)

        true_labels.extend(true_labels_batch)

true_labels = np.array(true_labels)
predicted_labels = np.array(predicted_labels)
image_idcs = np.array(image_idcs)
true_labels
predicted_labels<jupyter_output><empty_output><jupyter_text>Calculate Matthews correlation coefficientSee the [Wikipedia entry](https://en.wikipedia.org/wiki/Phi_coefficient).MCC is useful for a multiclass classification problem with highly imbalanced classes.<jupyter_code>from sklearn.metrics import matthews_corrcoef
mcc = matthews_corrcoef(true_labels, predicted_labels)
mcc<jupyter_output><empty_output><jupyter_text>Display images which were predicted incorrectly<jupyter_code>error_image_idcs = image_idcs[np.flatnonzero(predicted_labels != true_labels)]
error_image_idcs

error_predictions = predicted_labels[np.flatnonzero(predicted_labels != true_labels)]
from pybbbc import BBBC021

bbbc021 = BBBC021()
def make_layout(image_idx):
    image, metadata = bbbc021[error_image_idcs[image_idx]]

    predicted_class = dataset.label_to_class[error_predictions[image_idx]]

    prefix = f"{metadata.compound.compound}, {metadata.compound.moa}, Pred: {predicted_class}"

    plots = []

    cmaps = ["fire", "kg", "kb"]

    for channel_idx, im_channel in enumerate(image):
        plot = hv.Image(
            im_channel,
            bounds=(0, 0, im_channel.shape[1], im_channel.shape[0]),
            label=f"{prefix}",  # | {bbbc021.CHANNELS[channel_idx]}",
        ).opts(cmap=cmaps[channel_idx])
        plots.append(plot)

    plots.append(
        hv.RGB(
            image.transpose(1, 2, 0),
            bounds=(0, 0, im_channel.shape[1], im_channel.shape[0]),
            label="Channel overlay",
        )
    )

    return hv.Layout(plots).cols(2)


hv.DynamicMap(make_layout, kdims="image").redim.range(
    image=(0, len(error_image_idcs) - 1)
).opts(
    hv.opts.Image(frame_width=450, aspect="equal", active_tools=["wheel_zoom"]),
    hv.opts.RGB(frame_width=450, aspect="equal", active_tools=["wheel_zoom"]),
)<jupyter_output><empty_output><jupyter_text>Build and visualize confusion matrix  Metadata / MoA labels<jupyter_code>image_df = dataset.image_df
image_df
moa_df = dataset.moa_df
moa_df
all_moas = np.array(dm.all_dataset.moa_df.query('moa != "null"')['moa'].unique())
all_moas<jupyter_output><empty_output><jupyter_text>Find subset of MoAs not in test set<jupyter_code>moa_in_test = moa_df["moa"].unique()

missing_moas = set(all_moas).difference(moa_in_test)

print(f"MoAs not in test set: {missing_moas}")<jupyter_output><empty_output><jupyter_text>Construct and normalize confusion matrix<jupyter_code>import xarray as xr
import hvplot.xarray
cmat = xr.DataArray(
    confusion_matrix(
        true_labels,
        predicted_labels,
        labels=np.arange(
            np.array(list(dataset.class_to_label.values())).max() + 1
        ),
    ),
    dims=["moa_true", "moa_predicted"],
    coords=dict(moa_true=moas, moa_predicted=moas),
    name="confusion",
)

cmat
cmat_normed = (cmat / cmat.sum("moa_predicted")).pipe(
    lambda da: da.where(~da.isnull(), other=0)
)<jupyter_output><empty_output><jupyter_text>Visualize confusion matrix<jupyter_code>cmat_normed.copy().hvplot.heatmap(  # without copying, we can't unpickle later
    "moa_predicted",
    "moa_true",
    "confusion",
    rot=45,
    frame_width=300,
    frame_height=300,
    cmap="bjy",
    ylabel="True MoA",
    xlabel="Predicted MoA",
    title="Model predictions vs. true MoAs",
    clim=(0, 1),
)<jupyter_output><empty_output><jupyter_text>Dump results to be used later<jupyter_code>import pickle as pkl

with open("eval_results.pkl", "wb") as fhdl:
    pkl.dump(
        dict(
            model_name=model.__class__.__name__,
            mcc=mcc,
            cmat=cmat,
            cmat_normed=cmat_normed,
            image_df=image_df,
            moa_df=moa_df,
            true_labels=true_labels,
            predicted_labels=predicted_labels,
            image_idcs=image_idcs,
            error_image_idcs=error_image_idcs,
            error_predictions=error_predictions,
            error_classes=[
                dataset.label_to_class[error_predictions[image_idx]]
                for image_idx in range(len(error_predictions))
            ],
        ), fhdl
    )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10875258442453481, 'fraction_numerical': 0.011302549965541007, 'mean_word_length': 4.754163362410785, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '69118', 'license': 'MIT', 'n_tokens_mistral': 2601, 'n_tokens_neox': 2496, 'n_words': 482, 'path': 'notebooks/04_evaluate_model.ipynb', 'repo_name': 'zbarry/pytorch-hcs', 'size': '7255'}