<jupyter_start><jupyter_text>Analysis of the Faster RCNN model for grape detection Author: <NAME> (2021) **General information**This notebook allows to analyse the results obtained with a Faster RCNN model, trained on the AI4Agriculture (AI4EU project pilot) dataset.  **Running this notebook**This notebook can be run:1. from google Colaboratory 1. locally. When running locally, use a virtualenv according to [requirements.txt](requirements.txt). Configure and run the next four code cells according to the option selected (Colab or local).Before running this notebook, the faster RCNN model has to be trained using the script [train_unet.py](train_faster.py) in this repository.   **Notebook & environment setup**<jupyter_code># Run this cell only if you intend to work with the data stored in Google Drive
from google.colab import drive
drive.mount("/content/drive")
# Folder checkpoints/ contain the trained models. Folder data/ contains the images (data/completedataset/), the annotations (data/all_labels/) and the train/validation/test splits (data/splits/)

# If you want to work with the data in Google Drive (not on the Calcula servers), create a folder containing the checkpoints/ and data/ folders
CHECKPOINTS_DIR='/content/drive/MyDrive/Recerca/Projectes/AI4EU/grape_detection/pau/checkpoints'
DATA_DIR='/content/drive/MyDrive/Recerca/Projectes/AI4EU/grape_detection/pau'
# If working on the server, these folders should exist on your working directory:
#CHECKPOINTS_DIR='./checkpoints'
#DATA_DIR='.'
# Run this cell only if you intend to work with the data stored in Google Drive
!pip install dotmap
!pip install xmltodict
!pip install git+https://github.com/lucasb-eyer/pydensecrf.git
# Run this cell only if you intend to work with the data stored in Google Drive
!git clone https://github.com/imatge-upc/AI4Agriculture-grape-detection.git
#!git clone https://github.com/paumarquez/AI4Agriculture-grape-detection.git
%cd AI4Agriculture-grape-detection<jupyter_output><empty_output><jupyter_text>**Import libraries**<jupyter_code>from matplotlib import pyplot as plt
import numpy as np
from tqdm import tqdm
import sys
from collections import Counter
import pandas as pd
import matplotlib.patches as patches
import seaborn as sns
import copy
import torch
from dotmap import DotMap
import os
import json
import pickle
from itertools import chain
from torchvision.ops import box_iou
from src.metrics.recall_precision_ap import recall_precision_ap
from src.metrics.utils import get_metrics_per_threshold
from src.models import create_model
from src.data.data_loader import CreateDataLoader
from src.utils.mask_bbox_utils import apply_nms, get_matching_bboxes, get_wrong_bboxes # Added, JRMR
from src.utils.mask_bbox_utils import get_aspect_ratio # Added, JRMR<jupyter_output><empty_output><jupyter_text>**Create folders for results**<jupyter_code>if not os.path.isdir(f'{DATA_DIR}/results'):
    os.mkdir(f'{DATA_DIR}/results')
if not os.path.isdir(f'{DATA_DIR}/figures'):
    os.mkdir(f'{DATA_DIR}/figures')<jupyter_output><empty_output><jupyter_text>**Definitions and useful functions**<jupyter_code>def to_long_df(areas_list):
    names = [np.repeat(i, len(l)) for i, l in zip(['matched', 'almost_matched', 'not_matched'], areas_list)]
    df_areas = pd.DataFrame({"area":list(chain(*areas_list)), "key": list(chain(*names))})
    return df_areas
def get_options(checkpoint_dir):
    opt_dir     = os.path.join(checkpoint_dir, 'opt.json')
    weights_dir = os.path.join(checkpoint_dir, 'model_best.pt')

    with open(opt_dir, 'r') as fd:
        opt = json.load(fd)

    opt = DotMap({k: DATA_DIR + v[1:] if type(v) == str and v[:2] == './' else v for k, v in opt.items()})
    if "trainable_backbone_layers" not in opt:
        opt.trainable_backbone_layers = None
    # add checkpoint
    opt.checkpoint = weights_dir
    return opt
def get_dataset(split, data_opt, get='dataset'):
    dataset_opt = DotMap({
        **data_opt,
        "phase": split,
        "purpose": 'val',
        "batch_size": 1,
        "step_batch_size": 1
    })
    data_holder = CreateDataLoader(dataset_opt)
    if get   == 'dataset':
        return data_holder.dataset
    elif get == 'loader':
        return data_holder.dataloader
    elif get == 'holder':
        return data_holder
    else:
        raise BaseException('Incorrect get parameter')
from torch.utils.data import DataLoader
def my_collate(batch):
    data = [item['image'] for item in batch]
    target = [item['target'] for item in batch]
    return [data, target]

def get_image_data(image_id):
    if image_id in TRAIN_DATASET.ids:
        data, target = my_collate([TRAIN_DATASET.__getitem__(image_id)])
    elif image_id in VAL_DATASET.ids:
        data, target = my_collate([VAL_DATASET.__getitem__(image_id)])
    else:
        raise BaseException('wrong dataset')
    data   = [d.permute(2, 0, 1).to(device) for d in data]
    target = [{key : elem.to(device) if type(elem) == torch.Tensor else elem for key, elem in t.items()} for t in target]
    return data, target

def plot_image(image_id, model=None, boxes=None, show_gt=True, nms_threshold=0.0, title='', min_area_inference_filter=0,min_score_inference_filter=0):
    fig, ax = plt.subplots(figsize=(20,30))
    scores = None
    data, target = get_image_data(image_id)
    if model is not None:
        model.eval()
        with torch.no_grad():
            results = model(data)
        results = apply_nms(results, nms_threshold)[0]
        if title == '':
            title = get_metrics_per_threshold(
                target=[target],
                results=[results],
                score_threshold=min_score_inference_filter,
                mask_metrics=True,
                calc_ap=True
            )
    im = data[0].permute(1,2,0).cpu().numpy()
    ax.imshow(im)
    
    if model is not None:
        for j, bbox in enumerate(results['boxes'].cpu().numpy()):
            if results['scores'][j] > min_score_inference_filter and get_bbox_area(bbox) > min_area_inference_filter:
                rect = patches.Rectangle((bbox[0], bbox[1]), bbox[2]-bbox[0], bbox[3]-bbox[1], linewidth=2, edgecolor='white', facecolor='none')
                ax.add_patch(rect)
    if show_gt:
        for bbox in target[0]['boxes'].cpu().numpy():
            rect = patches.Rectangle((bbox[0], bbox[1]), bbox[2]-bbox[0], bbox[3]-bbox[1], linewidth=1, edgecolor='red', facecolor='none')
            ax.add_patch(rect)
    if boxes is not None:
        for bbox in (boxes if type(boxes[0]) != torch.Tensor else boxes.cpu().numpy()):
            rect = patches.Rectangle((bbox[0], bbox[1]), bbox[2]-bbox[0], bbox[3]-bbox[1], linewidth=2, edgecolor='blue', facecolor='none')
            ax.add_patch(rect)

    plt.title(title)
    fig.show()
def calc_metrics(model, split, data_opt, max_batches=100, nms_threshold=0.7, print_freq=20):
    model.eval()
    total_metrics = None
    n = 0
    loader = get_dataset(split, data_opt, 'loader')
    all_metrics = {}
    all_boxes   = {}
    for i, (data, target) in enumerate(loader):
        if i > max_batches:
            break
        if not i % print_freq:
            print(f"Iteration {i}")
        data   = [d.permute(2, 0, 1).to(device) for d in data]
        target = [{key : elem.to(device) if type(elem) == torch.Tensor else elem for key, elem in t.items()} for t in target]
        with torch.no_grad():
            results = model(data)
        results = apply_nms(results, nms_threshold)
        
        for t, r in zip(target,results):
            all_boxes[loader.dataset.ids[t["image_id"]]] = r["boxes"]

        res = pd.DataFrame([(th, *get_metrics_per_threshold(
            target=target,
            results=results,
            score_threshold=th,
            mask_metrics=True,
            calc_ap=th==0.9 # Only calc one, as it's independent from threshold
        )) for th in np.arange(0.0,1,0.1)], columns=[
            "threshold", "mask_recall", "mask_precision", "mask_ap", "mask_f1_score"
        ])
        for t in target:
            all_metrics[loader.dataset.ids[t["image_id"]]] = res
        if total_metrics is None:
            total_metrics = res.copy()
        else:
            total_metrics += res
        
        n += 1
    return all_metrics, total_metrics/n, all_boxes
def get_classified_gt_boxes(model, split, data_opt, max_batches=None):
    loader = get_dataset(split, data_opt, 'loader')
    not_matched    = {}
    matched        = {}
    almost_matched = {}
    ious = []
    for i, (data, target) in enumerate(loader):
        if max_batches and i >= max_batches:
            break
        data   = [d.permute(2, 0, 1).to(device) for d in data]
        target = [{key : elem.to(device) if type(elem) == torch.Tensor else elem for key, elem in t.items()} for t in target]
        with torch.no_grad():
            results = model(data)
        for t, r in zip(target, results):
            not_matched_curr    = get_matching_bboxes(t["boxes"], r["boxes"],lower_iou_thresh=0.0,  upper_iou_thresh=0.01)
            almost_matched_curr = get_matching_bboxes(t["boxes"], r["boxes"],lower_iou_thresh=0.01, upper_iou_thresh=0.7)
            matched_curr        = get_matching_bboxes(t["boxes"], r["boxes"],lower_iou_thresh=0.7,  upper_iou_thresh=1.0)
            not_matched_curr, almost_matched_curr, matched_curr = get_wrong_bboxes(t, r, lower_iou_thresh=0.1, upper_iou_thresh=0.7)
            not_matched[t["image_id_str"]]    = not_matched_curr.cpu().numpy().copy()
            almost_matched[t["image_id_str"]] = almost_matched_curr.cpu().numpy().copy()
            matched[t["image_id_str"]]        = matched_curr.cpu().numpy().copy()
            ious.append(box_iou(t["boxes"], r["boxes"]).max(1)[0].cpu().numpy().copy())
    return not_matched, almost_matched, matched, ious
models_to_compare = [
    'fasterRCNN_v7-3-3_2021-04-24 11:54:01', # 2nd best anchors (4 FPN)
]
models_to_compare = [os.path.join(CHECKPOINTS_DIR, run_name) for run_name in models_to_compare]
if torch.cuda.is_available():
    device = torch.device('cuda')
    print('GPU')
else:
    device = torch.device('cpu')
    print('CPU')
models_options = [get_options(checkpoint_dir) for checkpoint_dir in models_to_compare]
data_opt       = models_options[0]
print([m.version for m in models_options])
models = [create_model(opt).eval() for opt in models_options]
VAL_DATASET   = get_dataset('val',   data_opt, get='dataset')
TRAIN_DATASET = get_dataset('train', data_opt, get='dataset')
get_bbox_area = lambda bbox: (bbox[3] - bbox[1]) * (bbox[2] - bbox[0])<jupyter_output><empty_output><jupyter_text>**Raw data generation**During the first run, the Faster RCNN inferences for the validation & test sets will be generated and stored. On successive runs they will be read from the saved file, which is much faster.<jupyter_code>if not os.path.isfile(f'{DATA_DIR}/results/results_metrics_faster_definitive.pkl'):        
    all_metrics_val_m = []
    val_metrics_m     = []
    all_boxes_m       = []
    for i_split, split in enumerate(['val']):
        all_boxes_m.append([])
        val_metrics_m.append([])
        all_metrics_val_m.append([])
        
        for i, model in enumerate(models):
            model.to(device)
            all_metrics_val, val_metrics, all_boxes = calc_metrics(model, split, data_opt, nms_threshold=1, print_freq=10)
            all_metrics_val_m[i_split].append(all_metrics_val)
            val_metrics_m[i_split].append(val_metrics)
            all_boxes_m[i_split].append(all_boxes)
            model.to(torch.device("cpu"))
            print(f"Model {i} done")

    with open(f'{DATA_DIR}/results/results_metrics_faster_definitive.pkl', "wb") as fd:
        pickle.dump((models_options, all_boxes_m, val_metrics_m, all_metrics_val_m), fd)
else:
    with open(f'{DATA_DIR}/results/results_metrics_faster_definitive.pkl', "rb") as fd:
        models_options, all_boxes_m, val_metrics_m, all_metrics_val_m = pickle.load(fd)
val_metrics_m[0][0]
im_metrics = [(k, v.iloc[5]) for k, v in all_metrics_val_m[0][0].items()]
get_sorted_metrics = lambda metric: sorted(im_metrics, key=lambda kv: kv[1].mask_recall if metric == "recall" else (kv[1].mask_precision if metric == "precision" else (kv[1].mask_f1 if metric == "f1" else "qwe")))
get_sorted_metrics("recall")[:3]
[(m.loc[5].mask_f1_score, m.iloc[-1].mask_ap) for m in val_metrics_m[0]]
val_f_ap=[
    (opt.version, metrics.loc[5].mask_recall, metrics.loc[5].mask_precision, metrics.loc[5].mask_f1_score, metrics.iloc[-1].mask_ap)
        for metrics, opt in zip(val_metrics_m[0], models_options)
]
final_metrics = pd.DataFrame(val_f_ap, columns=['version', 'recall', 'precision', 'F-Score', 'AP'])
final_metrics
max_anchor_scale = pd.DataFrame([(128, ), (500, 4, False),*[(max(chain(*m.anchor_widths)), len(m.anchor_widths), max(m.anchor_ar[-1]) > 1.3) for m in models_options]])
''' TO REMOVE???
import pickle
with open(f'{DATA_DIR}/results/results_metrics_faster_definitive.pkl', "rb") as fd:   # JRMR!!!!
     models_options, all_boxes_m, val_metrics_m, all_metrics_val_m = pickle.load(fd)
'''
[m.version for m in models_options]
val_f_ap=[(opt.min_area, metrics.loc[5].mask_f1_score, metrics.iloc[-1].mask_ap) for metrics, opt in zip(val_metrics_m[0], models_options)]
min_area_res = pd.DataFrame(val_f_ap, columns = ['Minimum Area Threshold', "F-Score", "AP"]).sort_values('Minimum Area Threshold')
min_area_res
# Resize
resize_results = pd.DataFrame([('v7-1-2', 3.75, 0.6356, 0.7462),
('v7-1-3', 2.25, 0.66, 0.7691),
('v7-1-4', 1.5 ,0.7211, 0.8143),
('v7-1-5', 1.2, 0.7127, 0.8092)], columns = ['Version', 'Scale Factor', 'AP', 'F-Score'])
resize_results
nms_threshold = 0.4
for i, filename in enumerate(VAL_DATASET.ids):
    plot_image(filename, model=models[0].to(device), nms_threshold=nms_threshold,title='asd', show_gt=True)
    break<jupyter_output><empty_output><jupyter_text>Study non detected boxes (Recall)<jupyter_code>not_matched, almost_matched, matched, ious = get_classified_gt_boxes(models[0].to(device), 'test', data_opt)
all_matched_boxes        = list(chain(*matched.values()))
all_almost_matched_boxes = list(chain(*almost_matched.values()))
all_not_matched_boxes    = list(chain(*not_matched.values()))
matched_areas        = [get_bbox_area(box).item() for box in all_matched_boxes]
almost_matched_areas = [get_bbox_area(box).item() for box in all_almost_matched_boxes]
not_matched_areas    = [get_bbox_area(box).item() for box in all_not_matched_boxes]
''' TO REMOVE
def get_aspect_ratio(bbox):
    # Defined as torchivision -> height/width
    return (bbox[3] - bbox[1])/ (bbox[2] - bbox[0])
'''
matched_aspect_ratio        = [get_aspect_ratio(box).item() for box in all_matched_boxes]
almost_matched_aspect_ratio = [get_aspect_ratio(box).item() for box in all_almost_matched_boxes]
not_matched_aspect_ratio    = [get_aspect_ratio(box).item() for box in all_not_matched_boxes]
df_areas = to_long_df([matched_areas, almost_matched_areas, not_matched_areas])
bins=np.arange(df_areas["area"].min(), df_areas["area"].max(),2000)
len(all_matched_boxes)
print(f"#Didn't Match = {(df_areas['key'] == 'not_matched').sum()}\n" + \
f"#Almost Matched = {(df_areas['key'] == 'almost_matched').sum()}\n" + \
f"#Matched = {(df_areas['key'] == 'matched').sum()}")
fs = 14
fig, ax =plt.subplots(figsize=(7.5, 5))
sns.histplot(list(chain(*ious)),stat='probability',ax=ax,alpha=1, bins=40)

ax.set_xlabel('IoU', fontsize=fs)
ax.set_ylabel('Probability', fontsize=fs)
#plt.suptitle('Area Density Distribution', fontsize=20, y = 0.95)
plt.title('PDF of the IoU of GT with Predictions', fontsize=16)
plt.tight_layout()
#plt.savefig(os.path.join('figures', 'faster_boxes_iou_dist.png'), format='png',bbox_inches='tight')
plt.savefig(os.path.join(f'{DATA_DIR}/figures', 'faster_boxes_iou_dist.png'), format='png',bbox_inches='tight')
#plt.scale('log')
fig.show()
fig, ax =plt.subplots(figsize=(7.5, 5))
sns.histplot(df_areas[df_areas['key'] == 'not_matched'], x='area',stat='probability',ax=ax,color='red',alpha=0.5, label='No Match', bins=bins)
sns.histplot(df_areas[df_areas['key'] == 'almost_matched'], x='area',stat='probability', ax=ax, color='yellow',alpha=0.5, label='Unprecise Match', bins=bins)
sns.histplot(df_areas[df_areas['key'] == 'matched'], x='area',stat='probability', ax=ax, color='green',alpha=0.5, label='Match', bins=bins)

"""plt.annotate(
    f"#Didn't Match = {(df_areas['key'] == 'not_matched').sum()}\n" +
    f"#Almost Matched = {(df_areas['key'] == 'almost_matched').sum()}\n" +
    f"#Matched = {(df_areas['key'] == 'matched').sum()}",
    xy=(50000, 0.25), fontsize=14
)
"""
fs=14
ax.legend(fontsize=fs)
ax.set_xlabel('Area', fontsize=fs)
ax.set_ylabel('Probability', fontsize=fs)
#plt.suptitle('Area Density Distribution', fontsize=20, y = 0.95)
plt.title('PDF of the GT Area', fontsize=16)
plt.tight_layout()
#plt.savefig(os.path.join('figures', 'faster_boxes_match_dist.png'), format='png',bbox_inches='tight')
plt.savefig(os.path.join(f'{DATA_DIR}/figures', 'faster_boxes_match_dist.png'), format='png',bbox_inches='tight')
fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1414105940883816, 'fraction_numerical': 0.016154521510096578, 'mean_word_length': 4.252382416231171, 'pattern_counts': {'":': 6, '<': 26, '<?xml version=': 0, '>': 32, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '657665', 'license': 'MIT', 'n_tokens_mistral': 6160, 'n_tokens_neox': 5908, 'n_words': 1286, 'path': 'notebooks/faster_analysis.ipynb', 'repo_name': 'imatge-upc/AI4Agriculture-grape-detection', 'size': '17096'}