<jupyter_start><jupyter_text>Basic SetupSelect whether to use Google Colab or not.<jupyter_code># ONLY NECESSARY FOR LOCAL EXECUTION (WORKS WITHOUT THIS CELL IN GOOGLE COLAB)
# Setup that is necessary for jupyter notebook to find sibling-directories
# see: https://stackoverflow.com/questions/34478398/import-local-function-from-a-module-housed-in-another-directory-with-relative-im
    
import os
import sys
module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path:
    sys.path.append(module_path)

# Imports for this notebook

from data.data_util import triplets_as_batches, outputs_as_triplets
from util.solver import Solver
from data.triplet_dataset import Triplet_Dataset
from eval.evaluator import evaluate, eval_visibility_invariance, eval_transformation_illuminance_invariance
import torchvision.models as models
from torch.utils import data
from torch.utils.data.sampler import SubsetRandomSampler, SequentialSampler
import torchvision.transforms
import torch
import numpy as np

from eval.visualizer import Visualizer

from models.full_image_encoder import FullImageEncoder
from models.bounding_box_encoder import BoundingBoxEncoder
from models.reobj_encoder import REObjEncoder
from models.layers.rmac_layer import RMACLayer
from models.receptive_field_encoder import ReceptiveFieldEncoder
from models.losses.metrics import l2_dist_sum_weighted, cosine_similarity_sum
from models.losses.TripletLoss import TripletLoss
import torch.nn as nn
# Check training on GPU?

cuda = torch.cuda.is_available()

print("Training is on GPU with CUDA: {}".format(cuda))

device = "cuda:0" if cuda else "cpu"

print("Device: {}".format(device))

!nvidia-smi<jupyter_output><empty_output><jupyter_text>Load Data and ModelLoad dataset.Load the model for this notebook.<jupyter_code>from torchvision import transforms

# specify the path/to/3RScan dataset with subdirectories of scans and to the 2DInstances.txt file
eval_data_path = "/home/lukas/Desktop/datasets/3RScan/3RScan-10/all_in_one"

# output shape of the images
resize_image_shape = (224, 224) # (256, 256), None

# constructs the transform object
if resize_image_shape is not None:
    transform = transforms.Compose([
      Triplet_Dataset.rotate_vertical_transform,
      transforms.Resize(resize_image_shape),
      transforms.ToTensor()
    ])
else:
    transform = transforms.Compose([
      Triplet_Dataset.rotate_vertical_transform,
      transforms.ToTensor()
    ])

# specify dataset parameters here. This specifies how the Triplet_Dataset will be constructed
dataset_hparams = {
    "data_path": eval_data_path,
    "transform": str(transform),
    "neg_background_overlap_minimum": 256*256,
    "resize_image_shape": str(resize_image_shape),
    "sample_treshold_per_category": True,
    "sample_subset_percentage": 0.05,
    "sample_treshold": 99,
    "sample_fallback": True,
    "neg_background_overlap_minimum": 256*256,
    "bbox_data_aligned_vertically": True
}<jupyter_output><empty_output><jupyter_text>NETWORKS<jupyter_code>baselines = {   
    "reobj": REObjEncoder(),
    "bbox_crop_rmac_untrained": BoundingBoxEncoder(use_rmac_layer=True),
}

if torch.cuda.is_available():
    for model_name, model in baselines.items():
        baselines[model_name] = model.to("cuda:0")
    

networks = {
    "bbox_crop_rmac": BoundingBoxEncoder(use_rmac_layer=True),
    "bbox_crop_rmac_resnet": BoundingBoxEncoder(use_rmac_layer=True, encoder_type="ResNet"),
}

if torch.cuda.is_available():
    for model_name, model in networks.items():
        networks[model_name] = model.to("cuda:0")<jupyter_output><empty_output><jupyter_text>Load trained model weights<jupyter_code># Choose model into which the weights should be loaded
model = networks["bbox_crop_rmac"]
#model = baselines["bbox_crop_rmac_untrained"]
#model = baselines["reobj"]

modelname = "model_foo_trained_date" # for example: "model_2020-Aug-26_17-05-40"
print("Load model", modelname)

filepath = "../saved_models/" + modelname + ".pt"
model.load_state_dict(torch.load(filepath))<jupyter_output><empty_output><jupyter_text>TEST<jupyter_code># here we can specify what evaluations will be done
# all positive types will be evaluated for all negative types.
# each positive type and negative type together specify the configuration of a Triplet_Database.
# also see evaluator.py

eval_hparams = {
    "positive_types": {
        "all": [0.25, 0.25, 0.25, 0.25],
        "same": [1, 0, 0, 0],
        "other": [0, 1, 0, 0],
        "ambiguity": [0, 0, 1, 0],
        "movement": [0, 0, 0, 1]
    },
    "negative_types": {
        "easy": {
            "prob": [0.5, 0.5, 0, 0, 0],
            "samples": 99,
            "topk": [1,2,3,5,10,20,30,50,70,90]
        },
        "hard": {
            "prob": [0, 0, 0, 0.5, 0.5],
            "samples": 19,
            "topk": [1,2,3,5,10,15]
        },
        "mixed": {
            "prob": [0.2, 0.2, 0.2, 0.2, 0.2],
            "samples": 19,
            "topk": [1,2,3,5,10,15]
        }
    }
}

# here we can specify the details for an evaluation with the Invariace_Database
# also see evaluator.py

invariance_hparams = {
    "sample_treshold_per_anchor": 10,
    "render_size": 128,
    "bbox_data_aligned_vertically": dataset_hparams["bbox_data_aligned_vertically"], # needs to be copied currently
    "transformation_step": 0.1,
    "illuminance_step": 50
}

# for these models the evaluation will be done
print(models.keys())

# create output directory of evaluations
from pathlib import Path
out_dir = dataset_hparams["data_path"] + "/Evaluations"
Path(out_dir).mkdir(parents=True, exist_ok=True)
    
# verbose toggle
verbose = False
# evaluation without invariances
evaluate(models,
          transform,
          dataset_hparams["data_path"],
          custom_dataset_hparams=dataset_hparams,
          custom_eval_hparams=eval_hparams,
          verbose=verbose,
          out_dir=out_dir)


# evaluation with invariances
evaluate(models,
          transform,
          dataset_hparams["data_path"],
          custom_dataset_hparams=dataset_hparams,
          custom_eval_hparams=eval_hparams,
          custom_invariance_hparams=invariance_hparams,
          verbose=verbose,
          out_dir=out_dir)


# evaluation with visibility (without invariances)
eval_visibility_invariance(models,
                        transform,
                        dataset_hparams["data_path"],
                        custom_dataset_hparams=dataset_hparams,
                        custom_topk_hparams=eval_hparams,
                        custom_visibility_hparams={
                            "start": 0.6
                        },
                        verbose=verbose,
                        out_dir=out_dir)<jupyter_output><empty_output><jupyter_text>Evaluation: Visualization<jupyter_code># here we can specify the visualization hparams
# see evaluator.py

visualization_hparams = {
    "n": 2,
    "dim": -1,
    "p": [50],
    "color_mode": Visualizer.Color_Modes.image,
    "labels": []
}

# evaluate this again but this time show the visualizations as well
visualizers = evaluate(models,
                      transform,
                      dataset_hparams["data_path"],
                      custom_dataset_hparams=dataset_hparams,
                      custom_eval_hparams=eval_hparams,
                      verbose=verbose,
                      out_dir=out_dir)

print(visualizers)
# you can now select any visualizer from the visualizers dict and use it, e.g. :
# visualizer.visualize_pca(filters)
# also see visualizer.py<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09869375907111756, 'fraction_numerical': 0.021638738619870694, 'mean_word_length': 3.5552884615384617, 'pattern_counts': {'":': 44, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '542092', 'license': 'MIT', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2380, 'n_words': 546, 'path': 'notebooks/test.ipynb', 'repo_name': 'lukasHoel/3rscan-triplet-dataset-toolkit', 'size': '7579'}