<jupyter_start><jupyter_text>Init Learner and conf<jupyter_code>conf = get_config(training=False)
conf.batch_size=20 # Why bs_size can only be the number that divide 6000 well?
learner = face_learner(conf, inference=True)
def l2normalize(x, ord=2, axis=None):
    return x / np.linalg.norm(x, ord=ord, axis=axis)


def score_fn_original(feat1, feat2):
    return l2normalize(feat1).dot(l2normalize(feat2))
    

def score_fn_ours(feat_map1, feat_map2, learner, attention_strategy='learned', attention_weight=None):
    with torch.no_grad():
        assert len(feat_map1.shape) == 3  # [c, w, h]
        c, w, h = feat_map1.shape
        feat_map1 = torch.tensor(feat_map1).unsqueeze(0).to(conf.device)
        feat_map2 = torch.tensor(feat_map2).unsqueeze(0).to(conf.device)
        if attention_strategy == 'learned':
            attention = None
        elif attention_strategy == 'uniform':
            learner.model_attention.eval()
            attention = torch.ones([1, 1, w, h])  # [batch, c, w, h]
            attention /= attention.sum()
            attention = attention.to(conf.device)
        elif attention_strategy == 'fixed':
            assert attention_weight is not None
            assert attention_weight.shape[0] == w and attention_weight.shape[1] == h
            attention = torch.tensor(attention_weight).view(
                [1, 1, attention_weight.shape[0], attention_weight.shape[1]]
            ).type(feat_map1.type())
        else:
            raise NotImplementedError
        xCos, attention, cos_patched = learner.get_x_cosine(feat_map1, feat_map2, attention)
    return xCos.cpu().numpy()

def evaluate_and_plot(scores, is_sames, nrof_folds=10, dataset_name='AR Face', display_roc_curve=True):
    accuracy, best_threshold, roc_curve_tensor = learner.evaluate_and_plot_roc(scores, is_sames, nrof_folds=10)
    print(f'{dataset_name} - accuray:{accuracy:.5f} (1-{1-accuracy:.5f}), threshold:{best_threshold}')
    if display_roc_curve:
        display(trans.ToPILImage()(roc_curve_tensor))
    return accuracy<jupyter_output><empty_output><jupyter_text>Evaluate on ARFace<jupyter_code>from glob import glob
from copy import deepcopy
import random


class ARFaceSampler:
    def __init__(self, feat_root='./saved_features/ARFace/ir_se50_original/',
                 positive_num=6000, negative_num=6000,
                 conditions=('8', '9', '10', '11', '12', '13')):
        all_feature_files = sorted(glob(op.join(feat_root, '*.np*')))
        all_sample_names = [op.splitext(op.basename(f))[0] for f in all_feature_files]
        self.valid_sample_names = [n for n in all_sample_names if self._get_condition(n) in conditions]
        
        self.positive_pairs = self._build_pairs(num=positive_num, positive=True)
        self.negative_pairs = self._build_pairs(num=negative_num, positive=False)
        
    def _build_pairs(self, num, positive):
        # NOTE: this runs in O(n^2), but can be sped up
        pairs_left = random.choices(self.valid_sample_names, k=num)
        pairs_right = []
        for left in pairs_left:
            cid = self._get_candidate_id(left)
            available = [
                right
                for right in self.valid_sample_names
                if right != left and (self._get_candidate_id(left) == self._get_candidate_id(right)) == positive
            ]
            pairs_right.append(random.choice(available))
            
        pairs = [(left, right) for left, right in zip(pairs_left, pairs_right)]
        return pairs
            
    def _get_candidate_id(self, sample_name):
        return '-'.join(sample_name.split('-')[:2])
    
    def _get_condition(self, sample_name):
        return sample_name.split('-')[2]
    
    def __len__(self):
        return len(self.positive_pairs) + len(self.negative_pairs)
ARFace_sampler = ARFaceSampler(
    conditions=[f'{i}' for i in range(8, 14)] + [f'{i}' for i in range(21, 27)],
    positive_num=6000, negative_num=6000
)
# ARFace_sampler = ARFaceSampler(conditions=('1', '2', '3', '4'), positive_num=6000, negative_num=6000)
len(ARFace_sampler.valid_sample_names), len(ARFace_sampler)<jupyter_output><empty_output><jupyter_text>The original model<jupyter_code>all_scores = {}
all_acc = {}

def record_scores_and_acc(scores, acc, name, name2=None):
    all_scores[name] = scores
    all_acc[name] = acc
def _get_feature(feat_dir, suffix, ends_with='.npy'):
    return np.load(op.join(feat_dir, suffix) + ends_with)


feat_dir = './saved_features/ARFace/ir_se50_original/'
scores = []
for pair in ARFace_sampler.positive_pairs + ARFace_sampler.negative_pairs:
    p0_feat = _get_feature(feat_dir, pair[0]).squeeze(0)
    p1_feat = _get_feature(feat_dir, pair[1]).squeeze(0)
    scores.append(score_fn_original(p0_feat, p1_feat))
is_sames = np.array([True for _ in range(len(ARFace_sampler.positive_pairs))] +
                    [False for _ in range(len(ARFace_sampler.negative_pairs))])
scores = np.array(scores)
acc = evaluate_and_plot(scores, is_sames, nrof_folds=10)
record_scores_and_acc(scores, acc, name='original_ArcFace')
def evaluate_our_model_ARFace(model_name, score_functions, score_names,
                              pairs, is_sames):
    def _get_feature(feat_dir, suffix, ends_with='.npz'):
        return np.load(op.join(feat_dir, suffix) + ends_with)
    
    learner.load_state(conf, f'{model_name}.pth', model_only=True, from_save_folder=True, strict=True, model_atten=True)
    learner.model.eval()
    learner.model_attention.eval()
    
    feat_dir = f'./saved_features/ARFace/{model_name}/'
    scores_dict = {score_name: [] for score_name in score_names}
    for pair in pairs:
        p0_feat = _get_feature(feat_dir, pair[0])['feat_map'].squeeze(0)
        p1_feat = _get_feature(feat_dir, pair[1])['feat_map'].squeeze(0)
        for score_name, score_fn in zip(score_names, score_functions):
            scores_dict[score_name].append(score_fn(p0_feat, p1_feat))
    
    for score_name in score_names:
        print(f'---- Name: {score_name} ----')
        scores = np.array(scores_dict[score_name]).squeeze()
        acc = evaluate_and_plot(scores, is_sames, nrof_folds=10)
        record_scores_and_acc(scores, acc, name=f'{model_name}_{score_name}')


model_names = [
    '2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace',
    '2019-08-30-07-36_accuracy:0.9953333333333333_step:655047_None',
#     '2019-09-01-15-30_accuracy:0.9946666666666667_step:218346_CosFace',
#     '2019-08-25-14-35_accuracy:0.9931666666666666_step:218349_None',
]
score_names = [
    'uniform_weight', 'data_corr_stats', 'learned_weight', 
]
pairs = ARFace_sampler.positive_pairs + ARFace_sampler.negative_pairs
is_sames = np.array([True for _ in range(len(ARFace_sampler.positive_pairs))] +
                    [False for _ in range(len(ARFace_sampler.negative_pairs))])

is_sames = 
for model_name in model_names:
    fixed_weight = np.load(f'/tmp3/biolin/data/insightFace_pytorch/{model_name}.npy')
    fixed_weight /= fixed_weight.sum()
    score_functions = [
        lambda p0_feat, p1_feat: score_fn_ours(p0_feat, p1_feat, learner, attention_strategy='uniform'),
        lambda p0_feat, p1_feat: score_fn_ours(p0_feat, p1_feat, learner, attention_strategy='fixed',
                                               attention_weight=fixed_weight),
        lambda p0_feat, p1_feat: score_fn_ours(p0_feat, p1_feat, learner, attention_strategy='learned'),
        
    ]
    print(f'====== Evaluating on our model ({model_name}) ======')
    evaluate_our_model_ARFace(model_name, score_functions, score_names, pairs, is_sames)
    print('\n')
all_acc_ARFace = all_acc
all_scores_ARFace = all_scores
df = pd.DataFrame(all_scores)
to_plot_df = df.corr()[['original_ArcFace']]
to_plot_df = to_plot_df.rename(columns={'original_ArcFace': 'correlation'})
to_plot_df['accuracy'] = pd.DataFrame(all_acc, index=['accuracy']).T['accuracy']
to_plot_df
# sns.heatmap(to_plot_df, annot=True, fmt="f", )
df = pd.DataFrame(ARFace_sampler.positive_pairs + ARFace_sampler.negative_pairs)
df['is_same'] = np.array([True for _ in range(len(ARFace_sampler.positive_pairs))] +
                         [False for _ in range(len(ARFace_sampler.negative_pairs))])
df = df.rename(columns={0: 'face1', 1: 'face2'})
df
# df.to_csv("./saved_features/ARFace_pairs/1.csv")
to_plot_df = df.corr()[['original_ArcFace']]
to_plot_df = to_plot_df.rename(columns={'original_ArcFace': 'correlation'})
to_plot_df['accuracy'] = pd.DataFrame(all_acc, index=['accuracy']).T['accuracy']
to_plot_df
# sns.heatmap(to_plot_df, annot=True, fmt="f", )
df = pd.DataFrame(ARFace_sampler.positive_pairs + ARFace_sampler.negative_pairs)
df['is_same'] = np.array([True for _ in range(len(ARFace_sampler.positive_pairs))] +
                         [False for _ in range(len(ARFace_sampler.negative_pairs))])
df<jupyter_output><empty_output><jupyter_text>Evaluate on IJB-C<jupyter_code>loader = torch.utils.data.DataLoader(
    IJBCVerificationPathDataset('/tmp3/zhe2325138/IJB/IJB-C/', occlusion_lower_bound=1, leave_ratio=0.002),
    batch_size=1, shuffle=False
)
len(loader)
def _get_feature_original(feat_dir, suffixes, ends_with='.npy'):
    return np.concatenate([
        np.load(op.join(feat_dir, suffix[0]) + ends_with)
        for suffix in suffixes
    ], axis=0)


def _get_feature_ours(feat_dir, suffixes, ends_with='.npz'):
#     print(op.join(feat_dir, suffixes[0][0]) + ends_with)
    return np.concatenate([
        np.load(op.join(feat_dir, suffix[0]) + ends_with)['feat_map']
        for suffix in suffixes
    ], axis=0)


def run_IJBC_verification(loader, feat_dir, compare_strategy, _get_feature, score_fn, score_fn_kargs,
                          learner=None, attention_strategy=None):
    assert compare_strategy in ['compare_only_first_img', 'mean_comparison']
    is_sames = []
    scores = []
    init_time = time.time()
    ignored_count = 0
    for i, pair in enumerate(loader):
        if i % 1000 == 0:
            print(f"Processing match {i}, elapsed {time.time() - init_time:.1f} seconds")
        if len(pair['enroll_path_suffixes']) == 0 or len(pair['verif_path_suffixes']) == 0:
            ignored_count += 1
            continue

        if compare_strategy == 'compare_only_first_img':
            enroll_feature = _get_feature(feat_dir, pair['enroll_path_suffixes'][:1]).squeeze(0)
            verif_feature = _get_feature(feat_dir, pair['verif_path_suffixes'][:1]).squeeze(0)
        elif compare_strategy == 'mean_comparison':
            enroll_feature = _get_feature(feat_dir, pair['enroll_path_suffixes']).mean(axis=0)
            verif_feature = _get_feature(feat_dir, pair['verif_path_suffixes']).mean(axis=0)
        else:
            raise NotImplementedError
        score = score_fn(enroll_feature, verif_feature, **score_fn_kargs)
        scores.append(score)
        is_sames.append(pair['is_same'].cpu().numpy().astype(np.bool))
    
    print(f'{ignored_count} pairs are ignored since one of the template has no valid image.')
    scores = np.array(scores).squeeze()
    is_sames = np.array(is_sames).squeeze().astype(np.bool)
    return scores, is_sames
all_scores = {}
all_acc = {}

def record_scores_and_acc(scores, acc, name, name2=None):
    all_scores[name] = scores
    all_acc[name] = acc
scores, is_sames = run_IJBC_verification(
    loader, feat_dir='./saved_features/IJB-C/ir_se50_original/',
    score_fn=score_fn_original, _get_feature=_get_feature_original,
    compare_strategy='compare_only_first_img', score_fn_kargs={}
)
acc = evaluate_and_plot(scores, is_sames, nrof_folds=10, dataset_name='IJBC')
all_scores['original_ArcFace'] = scores
all_acc['original_ArcFace'] = acc
model_name = '2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace'
fixed_weight = np.load(f'/tmp3/biolin/data/insightFace_pytorch/{model_name}.npy')
fixed_weight /= fixed_weight.sum()

for attention_strategy in ['uniform', 'learned', 'fixed']:
    print(f'===== attention_strategy: {attention_strategy} =====')
    scores, is_sames = run_IJBC_verification(
        loader, feat_dir=f'./saved_features/IJB-C/{model_name}',
        score_fn=score_fn_ours, _get_feature=_get_feature_ours,
        compare_strategy='compare_only_first_img',
        score_fn_kargs={'learner': learner, 'attention_strategy': attention_strategy,
                        'attention_weight': fixed_weight}
    )
    acc = evaluate_and_plot(scores, is_sames, nrof_folds=10, dataset_name='IJBC')

    all_scores[f'{model_name}_{attention_strategy}'] = scores
    all_acc[f'{model_name}_{attention_strategy}'] = acc
all_scores
all_acc<jupyter_output><empty_output><jupyter_text>IJB-A<jupyter_code>all_scores = {}
all_acc = {}

def record_scores_and_acc(scores, acc, name, name2=None):
    all_scores[name] = scores
    all_acc[name] = acc
    
shuffle_order = np.arange(len(IJBAVerificationDataset()))
np.random.shuffle(shuffle_order)
'''
For IJB-A, I save features of t1 and t2 "for each comparison" and the corresponding label (is same or not)
'''
def _get_feature(fname):
    npz = np.load(fname)
    return npz['f1'], npz['f2'], npz['same']

def run_IJBA_verification(feat_dir, score_fn, score_fn_kargs, shuffle_order,
                          learner=None, attention_strategy=None, ):
    is_sames = []
    scores = []
    init_time = time.time()
    fnames = sorted(glob(op.join(feat_dir, '*.npz')))
    fnames = [fnames[i] for i in shuffle_order]
    print(len(fnames))
    for i, fname in enumerate(fnames):
        if i % 500 == 0:
            print(f"Processing match {i}, elapsed {time.time() - init_time:.1f} seconds")
        f1, f2, is_same = _get_feature(fname)
        score = score_fn(f1, f2, **score_fn_kargs)
        score = score.cpu().numpy() if torch.is_tensor(score) else score
        scores.append(score)
        is_sames.append(is_same.astype(np.bool))
    
    scores = np.array(scores).squeeze()
    is_sames = np.array(is_sames).squeeze().astype(np.bool)
    return scores, is_sames
scores, is_sames = run_IJBA_verification(
    feat_dir='./saved_features/IJB-A/split1/ir_se50/',
    score_fn=score_fn_original, score_fn_kargs={},
    shuffle_order=shuffle_order
)
print(np.histogram(scores), np.histogram(is_sames))
acc = evaluate_and_plot(scores, is_sames, nrof_folds=10, dataset_name='IJBC')
all_scores['original_ArcFace'] = scores
all_acc['original_ArcFace'] = acc
for far_target in [1e-1, 1e-2, 1e-3]:
    mean_tar = calculate_val_by_diff(
        thresholds='default', dist=(1 - scores), actual_issame=is_sames, far_target=0.001, ret_acc=True,
        nrof_folds=5
    )[0]
    print("TAR@FAR{far_target:.f}:{mean_tar}")
# calculate_val_far()
evaluate_and_plot_roc
model_name = '2019-09-02-08-21_accuracy:0.9968333333333333_step:436692_CosFace'
fixed_weight = np.load(f'/tmp3/biolin/data/insightFace_pytorch/{model_name}.npy')
fixed_weight /= fixed_weight.sum()

for attention_strategy in ['uniform', 'learned', 'fixed']:
    print(f'===== attention_strategy: {attention_strategy} =====')
    scores, is_sames = run_IJBA_verification(
        feat_dir=f'./saved_features/IJB-A/split1/{model_name}',
        score_fn=score_fn_ours,
        score_fn_kargs={'learner': learner, 'attention_strategy': attention_strategy,
                        'attention_weight': fixed_weight},
        shuffle_order=shuffle_order
    )
    acc = evaluate_and_plot(scores, is_sames, nrof_folds=10, dataset_name='IJBC')

    all_scores[f'{model_name}_{attention_strategy}'] = scores
    all_acc[f'{model_name}_{attention_strategy}'] = acc<jupyter_output>===== attention_strategy: uniform =====
11787
Processing match 0, elapsed 0.1 seconds
Processing match 500, elapsed 4.2 seconds
Processing match 1000, elapsed 8.6 seconds
Processing match 1500, elapsed 13.1 seconds
Processing match 2000, elapsed 17.4 seconds
Processing match 2500, elapsed 21.3 seconds
Processing match 3000, elapsed 25.3 seconds
Processing match 3500, elapsed 29.5 seconds
Processing match 4000, elapsed 34.4 seconds
Processing match 4500, elapsed 38.9 seconds
Processing match 5000, elapsed 43.3 seconds
Processing match 5500, elapsed 47.3 seconds
Processing match 6000, elapsed 52.1 seconds
Processing match 6500, elapsed 56.2 seconds
Processing match 7000, elapsed 59.9 seconds
Processing match 7500, elapsed 64.1 seconds
Processing match 8000, elapsed 68.5 seconds
Processing match 8500, elapsed 72.7 seconds
Processing match 9000, elapsed 76.9 seconds
Processing match 9500, elapsed 81.1 seconds
Processing match 10000, elapsed 85.8 seconds
Processing match 10500, elapsed 90.0[...]<jupyter_text>The following codes are just testing Evaluate on LFW (just testing)<jupyter_code>def getUnitAttention():
    x = torch.ones(conf.batch_size//2, 1, 7, 7).cuda()
    x /= x.flatten(2).sum(dim=2).repeat(1, 1, x.size(2) * x.size(3)).view_as(x)
    return x
unit_attention = getUnitAttention()
unit_attention.size()
lfw, lfw_issame = get_val_pair(conf.emore_folder, 'lfw')
learner.load_state(conf, 'ir_se50.pth', model_only=True, from_save_folder=True, strict=False, model_atten=False)
learner.model.eval()
learner.model.returnGrid = True  # Remember to reset this before return!
learner.model_attention.eval()<jupyter_output><empty_output><jupyter_text>Our models (wo/w attention)<jupyter_code># %%time
# accuracy, best_threshold, roc_curve_tensor = learner.evaluate_attention(
#     conf, lfw, lfw_issame, nrof_folds=10, tta=True, attention=unit_attention
# )
# print('lfw - accuray:{}, threshold:{}'.format(accuracy, best_threshold))
# trans.ToPILImage()(roc_curve_tensor)
# %%time
# accuracy, best_threshold, roc_curve_tensor = learner.evaluate_attention(
#     conf, lfw, lfw_issame, nrof_folds=10, tta=True, attention=None
# )
# print('lfw - accuray:{}, threshold:{}'.format(accuracy, best_threshold))
# trans.ToPILImage()(roc_curve_tensor)<jupyter_output><empty_output><jupyter_text>The original model<jupyter_code>%%time
learner.load_state(conf, 'ir_se50.pth', model_only=True, from_save_folder=True, strict=False, model_atten=False)
cosines = learner.get_original_cosines(lfw, conf, tta=True)
accuracy, best_threshold, roc_curve_tensor = learner.evaluate_and_plot_roc(cosines, lfw_issame, nrof_folds=10)
print('lfw - accuray:{}, threshold:{}'.format(accuracy, best_threshold))
trans.ToPILImage()(roc_curve_tensor)
calculate_val_by_diff(
    thresholds='default', dist=(1 - cosines), actual_issame=lfw_issame,
    far_target=0.001, ret_acc=True)
calculate_val_far((1 - 0.28250000000000103), (1 - cosines), lfw_issame, ret_acc=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12714795201556253, 'fraction_numerical': 0.03377283043337296, 'mean_word_length': 4.35813549507817, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 7, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '132020', 'license': 'MIT', 'n_tokens_mistral': 7039, 'n_tokens_neox': 6513, 'n_words': 1141, 'path': 'notebooks_evaluate/evaluate_models_on_IJBC_and_ARFace.ipynb', 'repo_name': 'ntubiolin/InsightFace_Pytorch', 'size': '18506'}