<jupyter_start><jupyter_text>Load pretrained detection model and Prediect directly  Load models of DNN and SVM<jupyter_code>cfg = get_cfg()
cfg.merge_from_file('configs/test_unfreeze_lastfews.yaml')
model = build_model(cfg)  # returns a torch.nn.Module

model.eval()
metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0])

# ckpt_file = 'checkpoints/coco/base_model/model_final.pth'
# ckpt_file = 'checkpoints/coco/faster_rcnn/30shot_person_freeze_last_cos/model_final.pth'
# ckpt_file = 'checkpoints/coco/faster_rcnn/30shot_person_freeze_last_fc/model_final.pth'
# ckpt_file = 'checkpoints/coco/faster_rcnn/30shot_person_unfreeze_lastfews/model_final.pth'
ckpt_file = 'checkpoints/coco/faster_rcnn/30shot_person_unfreeze_whole/model_0015999.pth'
# ckpt_file = 'checkpoints/coco/faster_rcnn/30shot_airplane_unfreeze_whole/model_0029999.pth'
DetectionCheckpointer(model).load(ckpt_file)

# clf = load('svm_results/svm_model_finetuned_prop_base.joblib') 
# clf = load('svm_results/svm_model_finetuned_prop_lastfew.joblib') 
# clf = load('svm_results/svm_model_finetuned_prop_whole.joblib') 
run_rcnn = False
ft_extractor_type = ckpt_file.split('/')[-2]
shots_num = 100
class_name = 'person'
file_name = 'svm_{}_{}_{}.joblib'.format(ft_extractor_type, shots_num, class_name)
clf = load('svm_results_0216/'+ file_name)

# define some functions for preprocessing
assert len(cfg.MODEL.PIXEL_MEAN) == len(cfg.MODEL.PIXEL_STD)
num_channels = len(cfg.MODEL.PIXEL_MEAN)
device = 'cuda'
pixel_mean = (
    torch.Tensor(cfg.MODEL.PIXEL_MEAN)
    .to(device)
    .view(num_channels, 1, 1)
)
pixel_std = (
    torch.Tensor(cfg.MODEL.PIXEL_STD)
    .to(device)
    .view(num_channels, 1, 1)
)
normalizer = lambda x: (x - pixel_mean) / pixel_std<jupyter_output><empty_output><jupyter_text>Register dataset<jupyter_code>from detectron2.data.datasets import register_coco_instances
json_dir = 'datasets/coco_experiments/seed1/full_box_30shot_person_trainval.json'
image_dir = 'datasets/coco/trainval2014'
register_coco_instances("30shot_person_train", {}, json_dir, image_dir)

json_dir = 'datasets/coco_experiments/seed1/full_box_{}shot_{}_test.json'.format(1000, class_name)
image_dir = 'datasets/coco/trainval2014'
testset_name = "{}_test".format(class_name)
register_coco_instances(testset_name, {}, json_dir, image_dir)

# json_dir = 'datasets/cocosplit/datasplit/5k.json'
# image_dir = 'datasets/coco/trainval2014'
# register_coco_instances("5k_test", {}, json_dir, image_dir)

# testset_name = "5k_test"<jupyter_output><empty_output><jupyter_text>Difference between some dataloader<jupyter_code># data_loader_train = build_detection_train_loader(cfg)
# data_loader_train_it = iter(data_loader_train)
# data = next(data_loader_train_it)
# print(len(data))
# print(data[0]['image'].shape)
# print(data)
# data_loader_test = build_detection_test_loader(cfg, "1000shot_person_test")
# data_loader_test_it = iter(data_loader_test)
data_loader_test = build_detection_test_loader(cfg, testset_name)
data_loader_test_it = iter(data_loader_test)
data = next(data_loader_test_it)
print(len(data))
print(data[0]['image'].shape)
print(len(data_loader_test))
# transform_gen = T.ResizeShortestEdge(
#             [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST],
#             cfg.INPUT.MAX_SIZE_TEST,
#             )
# print([cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST],
#             cfg.INPUT.MAX_SIZE_TEST)
# data_loader = build_detection_train_loader(DatasetCatalog.get("1000shot_person_test"),
#                                            mapper=DatasetMapper(cfg, is_train=False, augmentations=[transform_gen]),
#                                            total_batch_size = 10)
# data_loader_it = iter(data_loader)
# data = next(data_loader_it)
# print(len(data))
# print(data[0]['image'].shape)
# print(data)

## the type of dataloader is different 'AspectRatio Grouped Dataset' (don't know the reason) 
# run one iter
# evaluator = COCOEvaluator("1000shot_person_test", cfg, True, output_dir = cfg.OUTPUT_DIR)
# with torch.no_grad():
#     inputs = data
#     outputs = model(inputs)
#     evaluator.reset()
#     evaluator.process(inputs, outputs)
#     results = evaluator.evaluate()

# default test 
# run_rcnn = True
if run_rcnn:
    evaluator = COCOEvaluator(testset_name, cfg, True, output_dir = cfg.OUTPUT_DIR)
    evaluator.reset()
    inference_on_dataset(model, data_loader_test, evaluator)<jupyter_output><empty_output><jupyter_text>Run detector step by step (should have same results)<jupyter_code>if run_rcnn:
    evaluator = COCOEvaluator(testset_name, cfg, True, output_dir = cfg.OUTPUT_DIR)
    evaluator.reset()

    training = False
    save_results = False
    box2box_transform = Box2BoxTransform(
                weights=cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_WEIGHTS
            )
    smooth_l1_beta = cfg.MODEL.ROI_BOX_HEAD.SMOOTH_L1_BETA
    # test_score_thresh        = cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST
    test_score_thresh        = 0.5
    test_nms_thresh          = cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST
    test_detections_per_img  = cfg.TEST.DETECTIONS_PER_IMAGE


    with torch.no_grad():
        for idx, inputs in enumerate(data_loader_test):
            #     batched_inputs = data
            batched_inputs = inputs
            ###################################    
            #     outputs = model(inputs)     #
            #---------------------------------# 

            # Normalize, pad and batch the input images. (Preprocess_image)
            images = [x["image"].to('cuda') for x in batched_inputs]
            images = [normalizer(x) for x in images]
            images = ImageList.from_tensors(
                images, model.backbone.size_divisibility
            )

            # forward
            features = model.backbone(images.tensor)
    #         print('features shape:', features['p3'].shape)
            proposals, _ = model.proposal_generator(images, features)
    #         print('proposal num per img:', proposals_list[0].objectness_logits.shape)


            #     results, _ = model.roi_heads(images, features, proposals)
            #     print('\ninstance for image 0:', results[0], '\n')

            # run roi_heads step by step
            if training:
            #         proposals = [proposal for proposal in proposals]
                targets = [d['instances'].to('cuda') for d in data]
                proposals = model.roi_heads.label_and_sample_proposals(proposals, targets)

            box_features = model.roi_heads.box_pooler(
                [features[f] for f in ["p2", "p3", "p4", "p5"]], [x.proposal_boxes for x in proposals]
            )
    #         print(box_features.shape)
            box_features = model.roi_heads.box_head(box_features)
    #         print(box_features.shape)

            pred_class_logits, pred_proposal_deltas = model.roi_heads.box_predictor(
                box_features
            )
    #         print('pred_class_logits', pred_class_logits[:3])
    #         print('pred_proposal_deltas', pred_proposal_deltas.shape)

            outputs = FastRCNNOutputs(
                box2box_transform,
                pred_class_logits,
                pred_proposal_deltas,
                proposals,
                smooth_l1_beta,
            )

            results, _ = outputs.inference(
                    test_score_thresh,
                    test_nms_thresh,
                    test_detections_per_img,
            )

            # postprocess: resize images
            processed_results = []
            for results_per_image, input_per_image, image_size in zip(
                results, batched_inputs, images.image_sizes
            ):
                height = input_per_image.get("height", image_size[0])
                width = input_per_image.get("width", image_size[1])
                r = detector_postprocess(results_per_image, height, width)
                processed_results.append({"instances": r})
    #         print('postprocessed instance for image 0:\n', processed_results[0], '\n')
    ###################################  

            # evaluate
            evaluator.process(inputs, processed_results)
            save_results = True
            if save_results:
                # visualizer
                # inputs should be only one image
                raw_image = cv2.imread(batched_inputs[0]['file_name'])
                result_show = processed_results[0]["instances"]
                v = Visualizer(raw_image,
                                metadata=MetadataCatalog.get(testset_name), 
                                scale=1.0, 
                                instance_mode=ColorMode.IMAGE   # remove the colors of unsegmented pixels
                    )
                v = v.draw_instance_predictions(result_show.to("cpu"))

                folder_name = './test_0216/det_last_fc_0.5/'
                os.makedirs(folder_name, exist_ok=True)
                det_img_dir = folder_name + str(idx) + '.jpg'
                cv2.imwrite(det_img_dir, v.get_image())
        eval_results = evaluator.evaluate()<jupyter_output><empty_output><jupyter_text>Add SVM classifier as the final layer<jupyter_code>evaluator = COCOEvaluator(testset_name, cfg, True, output_dir = cfg.OUTPUT_DIR)
evaluator.reset()
training = False
box2box_transform = Box2BoxTransform(
            weights=cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_WEIGHTS
        )
smooth_l1_beta = cfg.MODEL.ROI_BOX_HEAD.SMOOTH_L1_BETA
# test_score_thresh        = cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST
test_score_thresh        = 0.5
test_nms_thresh          = cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST
test_detections_per_img  = cfg.TEST.DETECTIONS_PER_IMAGE
print(test_score_thresh, test_nms_thresh, test_detections_per_img)

with torch.no_grad():
    for idx, inputs in enumerate(data_loader_test):
        #     batched_inputs = data
        batched_inputs = inputs
        ###################################    
        #     outputs = model(inputs)     #
        #---------------------------------# 

        # Normalize, pad and batch the input images. (Preprocess_image)
        images = [x["image"].to('cuda') for x in batched_inputs]
        images = [normalizer(x) for x in images]
        images = ImageList.from_tensors(
            images, model.backbone.size_divisibility
        )

        # forward
        features = model.backbone(images.tensor)
#         print('features shape:', features['p3'].shape)
        proposals, _ = model.proposal_generator(images, features)
#         print('proposal num per img:', proposals[0].objectness_logits.shape) # 1000 proposals


        # run roi_heads step by step
        if training:
    #         proposals = [proposal for proposal in proposals]
            targets = [x['instances'].to('cuda') for x in batched_inputs]
            proposals = model.roi_heads.label_and_sample_proposals(proposals, targets)

        box_features = model.roi_heads.box_pooler(
            [features[f] for f in ["p2", "p3", "p4", "p5"]], [x.proposal_boxes for x in proposals]
        )
#         print(box_features.shape)
        box_features = model.roi_heads.box_head(box_features)
#         print(box_features.shape)

        pred_class_logits, pred_proposal_deltas = model.roi_heads.box_predictor(
            box_features
        )
#         print('pred_class_logits', pred_class_logits[:3])
#         print('pred_proposal_deltas', pred_proposal_deltas.shape)

        outputs = FastRCNNOutputs(
            box2box_transform,
            pred_class_logits,
            pred_proposal_deltas,
            proposals,
            smooth_l1_beta,
        )

        results, _ = outputs.inference(
                test_score_thresh,
                test_nms_thresh,
                test_detections_per_img,
        )

        # postprocess: resize images
        processed_results = []
        for results_per_image, input_per_image, image_size in zip(
            results, batched_inputs, images.image_sizes
        ):
            height = input_per_image.get("height", image_size[0])
            width = input_per_image.get("width", image_size[1])
            r = detector_postprocess(results_per_image, height, width)
            processed_results.append({"instances": r})
#         print('postprocessed instance for image 0:\n', processed_results[0], '\n')

        # SVM 
        X = box_features.to('cpu').detach().numpy()
#         y_hat = clf.predict(X)
        pred_class_logits_svm = clf.predict_log_proba(X)
        pred_class_logits_svm = torch.from_numpy(pred_class_logits_svm).to('cuda')
#         print(y_hat.shape)
#         print(pred_class_logits_svm[:3])

        outputs_svm = FastRCNNOutputs(
            box2box_transform,
            pred_class_logits_svm,
            pred_proposal_deltas,
            proposals,
            smooth_l1_beta,
        )

        pred_instances_svm, _ = outputs_svm.inference(
                test_score_thresh,
                test_nms_thresh,
                test_detections_per_img,
        )

        processed_results_svm = []
        for results_per_image, input_per_image, image_size in zip(
            pred_instances_svm, batched_inputs, images.image_sizes
        ):
            height = input_per_image.get("height", image_size[0])
            width = input_per_image.get("width", image_size[1])
            r = detector_postprocess(results_per_image, height, width)
            processed_results_svm.append({"instances": r})
#         print('\n\nSVM postprocessed instance for image 0:\n', processed_results_svm[0], '\n')  
    
        ###################################  
        
        # evaluate
        evaluator.process(inputs, processed_results_svm)
        save_results = False
        if save_results:
            # visualizer
            # inputs should be only one image
            raw_image = cv2.imread(batched_inputs[0]['file_name'])
            result_show = processed_results_svm[0]["instances"]
            v = Visualizer(raw_image,
                            metadata=MetadataCatalog.get("1000shot_person_test"), 
                            scale=1.0, 
                            instance_mode=ColorMode.IMAGE   # remove the colors of unsegmented pixels
                )
            v = v.draw_instance_predictions(result_show.to("cpu"))
        
#             det_img_folder = time.strftime("%d_%H_%M/")
            folder_name = './test_0216/svm_{}_{}_{}_{}/'.format(ft_extractor_type, shots_num, class_name, test_score_thresh)
            os.makedirs(folder_name, exist_ok=True)
            det_img_dir = folder_name + str(idx) + '.jpg'
            cv2.imwrite(det_img_dir, v.get_image())
    results = evaluator.evaluate()<jupyter_output>0.5 0.5 100
{'fraction_non_alphanumeric': 0.11183044841215756, 'fraction_numerical': 0.01117623006678479, 'mean_word_length': 2.6733416770963703, 'pattern_counts': {'":': 3, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '590101', 'license': 'Apache-2.0', 'n_tokens_mistral': 4865, 'n_tokens_neox': 4709, 'n_words': 724, 'path': 'FRCNN-SVM-Eval.ipynb', 'repo_name': 'CanboYe/few-shot-object-detection', 'size': '14674'}