<jupyter_start><jupyter_text>Init Learner and conf<jupyter_code>conf = get_config(training=False)
conf.batch_size=20 # Why bs_size can only be the number that divide 6000 well?
learner = face_learner(conf, inference=True)<jupyter_output>{'fixed_str': 'ir_se50.pth', 'pretrainedMdl': 'ir_se50.pth', 'data_path': PosixPath('data'), 'work_path': PosixPath('work_space'), 'model_path': PosixPath('work_space/models'), 'log_path': PosixPath('work_space/log'), 'save_path': PosixPath('work_space/save'), 'exp_title': 'xCos', 'exp_comment': 'expMS1M', 'input_size': [112, 112], 'embedding_size': 1568, 'use_mobilfacenet': False, 'net_depth': 50, 'drop_ratio': 0.6, 'net_mode': 'ir_se', 'device': device(type='cuda', index=0), 'test_transform': Compose(
    ToTensor()
    Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
), 'data_mode': 'emore', 'vgg_folder': PosixPath('data/faces_vgg_112x112'), 'ms1m_folder': PosixPath('data/faces_ms1m_112x112'), 'emore_folder': PosixPath('data/faces_emore'), 'batch_size': 20, 'USE_SOFTMAX': True, 'SOFTMAX_T': 1, 'facebank_path': PosixPath('data/facebank'), 'threshold': 1.5, 'threshold_xCos': 0.2338, 'face_limit': 10, 'min_face_size': 30}
ir_se_50 model generated<jupyter_text>Process IJB-C data and save them<jupyter_code>loader = torch.utils.data.DataLoader(
    IJBCAllCroppedFacesDataset('/tmp3/zhe2325138/IJB/IJB-C/'),
    batch_size=1
)<jupyter_output><empty_output><jupyter_text>Original model<jupyter_code>learner.load_state(conf, 'ir_se50.pth', model_only=True, from_save_folder=True, strict=False, model_atten=False)
learner.model.eval()
learner.model.returnGrid = True  # Remember to reset this before return!
learner.model_attention.eval()

dst_dir = './saved_features/IJB-C/ir_se50_original/'
os.makedirs(op.join(dst_dir, 'img'), exist_ok=True)
os.makedirs(op.join(dst_dir, 'frames'), exist_ok=True)

with torch.no_grad():
    for i, batch in enumerate(loader):
        tensor = batch['tensor']
        src_path = batch['path']
        if i % 200 == 0:
            print(f'Processing {src_path[0]}')
        target_path = op.join(dst_dir, *src_path[0].split('/')[-2:]) + '.npy'
        if op.exists(target_path):
            if i % 200 == 0:
                print(f"Skipping {target_path} because it exists.")
            continue
        feat = process_batch_original(tensor, tta=True)
        np.save(target_path, feat.cpu().numpy())<jupyter_output><empty_output><jupyter_text>Our model<jupyter_code>def run_ours_IJBC(model_name):
    learner.load_state(
        conf, f'{model_name}.pth',
        model_only=True, from_save_folder=True, strict=True, model_atten=True)
    learner.model.eval()
    learner.model.returnGrid = True  # Remember to reset this before return!
    learner.model_attention.eval()

    dst_dir = f'./saved_features/IJB-C/{model_name}/'
    os.makedirs(op.join(dst_dir, 'img'), exist_ok=True)
    os.makedirs(op.join(dst_dir, 'frames'), exist_ok=True)

    with torch.no_grad():
        for i, batch in enumerate(loader):
            tensor = batch['tensor']
            src_path = batch['path']
            target_path = op.join(dst_dir, *src_path[0].split('/')[-2:]) + '.npz'
            if i % 2000 == 0:
                print(f'Processing {src_path[0]} \nSaving to {target_path}')
            if op.exists(target_path):
                if i % 2000 == 0:
                    print(f"Skipping {target_path} because it exists.")
                continue
            flattened_feature, feat_map = process_batch_xcos(tensor, tta=True)
            np.savez(target_path, flattened_feature=flattened_feature.cpu().numpy(),
                     feat_map=feat_map.cpu().numpy())


model_names = [
#     '2019-09-01-15-30_accuracy:0.9946666666666667_step:218346_CosFace',
    '2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace',
#     '2019-08-25-14-35_accuracy:0.9931666666666666_step:218349_None',
    '2019-08-30-07-36_accuracy:0.9953333333333333_step:655047_None'
]
for model_name in model_names:
    print(f'-------- Runing on model {model_name} --------')
    run_ours_IJBC(model_name)<jupyter_output>-------- Runing on model {model_name} --------
Processing /tmp3/zhe2325138/IJB/IJB-C/cropped_faces/img/10001_101093.jpg 
Saving to ./saved_features/IJB-C/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/img/10001_101093.jpg.npz
Skipping ./saved_features/IJB-C/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/img/10001_101093.jpg.npz because it exists.
Processing /tmp3/zhe2325138/IJB/IJB-C/cropped_faces/img/12624_114173.jpg 
Saving to ./saved_features/IJB-C/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/img/12624_114173.jpg.npz
Skipping ./saved_features/IJB-C/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/img/12624_114173.jpg.npz because it exists.
Processing /tmp3/zhe2325138/IJB/IJB-C/cropped_faces/img/13041_117728.jpg 
Saving to ./saved_features/IJB-C/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/img/13041_117728.jpg.npz
Skipping ./saved_features/IJB-C/2019-09-02-08-21_accuracy:0.9968333333333333_ste[...]<jupyter_text>IJB-A<jupyter_code>loader_IJBA = torch.utils.data.DataLoader(
    IJBAVerificationDataset(only_first_image=False),
    batch_size=1
)
len(loader_IJBA)
def run_ours_IJBA(loader, model_name, xCos_or_original='xCos', only_first_image=True):
    if not only_first_image:
        assert loader.batch_size == 1
        
    model_atten = True if xCos_or_original == 'xCos' else False
    strict = True if xCos_or_original == 'xCos' else False
    learner.load_state(
        conf, f'{model_name}.pth',
        model_only=True, from_save_folder=True, strict=strict, model_atten=model_atten)
    learner.model.eval()
    learner.model.returnGrid = True  # Remember to reset this before return!
    learner.model_attention.eval()

    dst_dir = f'./saved_features/IJB-A_full_template/split1/{model_name}/'
    os.makedirs(dst_dir, exist_ok=True)
    begin_time = time.time()
    with torch.no_grad():
        for i, batch in enumerate(loader):
            if i % 50 == 0:
                print(f"Elapsed {time.time() - begin_time:.1f} seconds.")
            t1_tensors = batch['t1_tensors']
            t2_tensors = batch['t2_tensors']
            
            if not only_first_image:
                t1_tensors, t2_tensors = t2_tensors.squeeze(0), t2_tensors.squeeze(0)
                
            comparison_idx = batch['comparison_idx']
            is_same = batch['is_same']
            
            if xCos_or_original == 'xCos':
                _, feat1 = process_batch_xcos(t1_tensors, tta=True)
                _, feat2 = process_batch_xcos(t2_tensors, tta=True)
            elif xCos_or_original == 'original':
                feat1 = process_batch_original(t1_tensors, tta=True)
                feat2 = process_batch_original(t2_tensors, tta=True)
            else:
                raise NotImplementedError
                
            if not only_first_image:
                feat1, feat2 = feat1.unsqueeze(0), feat2.unsqueeze(0
                                                                  )
#             print(feat1.shape, t1_tensors.shape)

            for idx, f1, f2, same in zip(comparison_idx, feat1, feat2, is_same):
                target_path = op.join(dst_dir, str(int(idx.cpu().numpy()))) + '.npz'
                if op.exists(target_path):
                    if i % 50 == 0:
                        print(f"Skipping {target_path} because it exists.")
                    continue
                if i % 50 == 0:
                    print(f'Saving to {target_path}')
                    
                np.savez(target_path, idx=idx.cpu().numpy(), same=same.cpu().numpy(),
                         f1=f1.cpu().numpy(), f2=f2.cpu().numpy(), )
run_ours_IJBA(loader_IJBA, 'ir_se50', xCos_or_original='original', only_first_image=False)
model_names = [
    '2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace',
    '2019-08-30-07-36_accuracy:0.9953333333333333_step:655047_None',
#     '2019-09-01-15-30_accuracy:0.9946666666666667_step:218346_CosFace',
#     '2019-08-25-14-35_accuracy:0.9931666666666666_step:218349_None',
]
for model_name in model_names:
    print(f'-------- Runing on model {model_name} --------')
    run_ours_IJBA(loader_IJBA, model_name, xCos_or_original='xCos', only_first_image=False)<jupyter_output>-------- Runing on model 2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace --------
Loading attention module from work_space/save/model_attention_2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace.pth
Skipping ./saved_features/IJB-A/split1/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/0.npz because it exists.
Skipping ./saved_features/IJB-A/split1/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/1.npz because it exists.
Skipping ./saved_features/IJB-A/split1/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/2.npz because it exists.
Skipping ./saved_features/IJB-A/split1/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/3.npz because it exists.
Skipping ./saved_features/IJB-A/split1/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/4.npz because it exists.
Skipping ./saved_features/IJB-A/split1/2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace/5.npz because it exis[...]<jupyter_text>Process ARFace data and save them<jupyter_code>loader = torch.utils.data.DataLoader(ARVerificationAllPathDataset(), batch_size=1)
learner.load_state(conf, 'ir_se50.pth', model_only=True, from_save_folder=True, strict=False, model_atten=False)
learner.model.eval()
learner.model.returnGrid = True  # Remember to reset this before return!
learner.model_attention.eval()

dst_dir = './saved_features/ARFace/ir_se50_original/'
os.makedirs(dst_dir, exist_ok=True)
with torch.no_grad():
    for i, batch in enumerate(loader):
        tensor = batch['image_tensor']
        fname = batch['fname'][0]
        target_path = op.join(dst_dir, fname) + '.npy'
        if i % 200 == 0:
            print(f'Processing {fname}, saving to {target_path}')
        if op.exists(target_path):
            if i % 200 == 0:
                print(f"Skipping {target_path} because it exists.")
            continue
        feat = process_batch_original(tensor, tta=True)
        np.save(target_path, feat.cpu().numpy())
def compute_our_model(name):
    learner.load_state(conf, f'{name}.pth', model_only=True, from_save_folder=True, strict=True, model_atten=True)
    learner.model.eval()
    learner.model.returnGrid = True  # Remember to reset this before return!
    learner.model_attention.eval()

    dst_dir = f'./saved_features/ARFace/{name}/'
    os.makedirs(dst_dir, exist_ok=True)
    with torch.no_grad():
        for i, batch in enumerate(loader):
            tensor = batch['image_tensor']
            fname = batch['fname'][0]
            target_path = op.join(dst_dir, fname) + '.npz'
            if i % 200 == 0:
                print(f'Processing {fname}, saving to {target_path}')
            if op.exists(target_path):
                if i % 200 == 0:
                    print(f"Skipping {target_path} because it exists.")
                continue
            flattened_feature, feat_map = process_batch_xcos(tensor, tta=True)
            np.savez(target_path, flattened_feature=flattened_feature.cpu().numpy(),
                     feat_map=feat_map.cpu().numpy())
            
names = [
    '2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace',
    '2019-08-25-14-35_accuracy:0.9931666666666666_step:218349_None',
    '2019-08-30-07-36_accuracy:0.9953333333333333_step:655047_None',
#     '2019-09-01-15-30_accuracy:0.9946666666666667_step:218346_CosFace',
]
for name in names:
    print(f'---------------------- Testing model {name} ----------------------')
    compute_our_model(name)
1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13866286385370355, 'fraction_numerical': 0.10116600956295613, 'mean_word_length': 3.2931220741807707, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 26, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '657764', 'license': 'MIT', 'n_tokens_mistral': 5108, 'n_tokens_neox': 4270, 'n_words': 635, 'path': 'notebooks_evaluate/run_models_on_IJBC_and_ARFace.ipynb', 'repo_name': 'ntubiolin/InsightFace_Pytorch', 'size': '11921'}