def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix='', lang='en', lang2id=None, print_result=True, adapter_weight=None, lang_adapter_names=None, task_name=None, calc_weight_step=0):
    eval_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode=mode, lang=lang, lang2id=lang2id)
    args.eval_batch_size = (args.per_gpu_eval_batch_size * max(1, args.n_gpu))
    if args.get_attr:
        eval_sampler = (RandomSampler(eval_dataset) if (args.local_rank == (- 1)) else DistributedSampler(eval_dataset))
    else:
        eval_sampler = (SequentialSampler(eval_dataset) if (args.local_rank == (- 1)) else DistributedSampler(eval_dataset))
    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
    if (args.n_gpu > 1):
        model = torch.nn.DataParallel(model)
    logger.info(('***** Running evaluation %s in %s *****' % (prefix, lang)))
    logger.info('  Num examples = %d', len(eval_dataset))
    logger.info('  Batch size = %d', args.eval_batch_size)
    eval_loss = 0.0
    nb_eval_steps = 0
    preds = None
    out_label_ids = None
    model.eval()
    counter = 0
    head_importances = None
    all_head_importances = None
    for batch in tqdm(eval_dataloader, desc='Evaluating'):
        counter += 1
        logger.info(f'Batch number = {counter}')
        batch = tuple((t.to(args.device) for t in batch))
        if (calc_weight_step > 0):
            adapter_weight = calc_weight_multi(args, model, batch, lang_adapter_names, task_name, adapter_weight, calc_weight_step, lang=lang)
        if args.get_attr:
            inputs = {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3], 'adapter_weights': adapter_weight}
            if (args.model_type != 'distilbert'):
                inputs['token_type_ids'] = (batch[2] if (args.model_type in ['bert', 'xlnet']) else None)
            if (args.model_type == 'xlm'):
                inputs['langs'] = batch[4]
            inputs['output_attentions'] = True
            outputs = model(**inputs)
            (tmp_eval_loss, logits, attentions, kept_labels, kl_logits) = outputs
            attr_all = []
            res_attr = []
            input_len = int(inputs['attention_mask'][0].sum())
            example_head_importances = None
            logits = logits[0]
            for tar_layer in range(12):
                att = attentions[tar_layer][0]
                pred_labels = torch.argmax(logits, dim=(- 1))
                (scale_att, step) = scaled_input(att.data)
                scale_att.requires_grad_(True)
                attr_all = None
                prob_all = None
                for j_batch in range(1):
                    one_batch_att = scale_att[(j_batch * 16):((j_batch + 1) * 16)]
                    (_, grad) = model(input_ids=inputs['input_ids'], token_type_ids=inputs['token_type_ids'], attention_mask=inputs['attention_mask'], labels=inputs['labels'], tar_layer=tar_layer, tmp_score=one_batch_att, pred_labels=pred_labels)
                    grad = grad.sum(dim=0)
                    attr_all = (grad if (attr_all is None) else torch.add(attr_all, grad))
                attr_all = (attr_all[:, 0:input_len, 0:input_len] * step[:, 0:input_len, 0:input_len])
                if (example_head_importances is None):
                    example_head_importances = torch.amax(attr_all, dim=(1, 2)).unsqueeze(0)
                else:
                    tmp = torch.amax(attr_all, dim=(1, 2))
                    tmp = tmp.unsqueeze(0)
                    example_head_importances = torch.cat((example_head_importances, tmp), dim=0)
                res_attr.append(attr_all.data)
            all_head_importances = (example_head_importances.unsqueeze(0) if (all_head_importances is None) else torch.cat((all_head_importances, example_head_importances.unsqueeze(0)), dim=0))
            head_importances = (example_head_importances if (head_importances is None) else torch.add(head_importances, example_head_importances))
            if (counter == 100):
                break
            continue
        with torch.no_grad():
            inputs = {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3], 'adapter_weights': adapter_weight}
            if (args.model_type != 'distilbert'):
                inputs['token_type_ids'] = (batch[2] if (args.model_type in ['bert', 'xlnet']) else None)
            if (args.model_type == 'xlm'):
                inputs['langs'] = batch[4]
            outputs = model(**inputs)
            (tmp_eval_loss, logits) = outputs[:2]
            if (args.n_gpu > 1):
                tmp_eval_loss = tmp_eval_loss.mean()
            eval_loss += tmp_eval_loss.item()
        nb_eval_steps += 1
        if (preds is None):
            preds = logits.detach().cpu().numpy()
            out_label_ids = inputs['labels'].detach().cpu().numpy()
        else:
            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
            out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0)
    if args.get_attr:
        head_importances = (head_importances / counter)
        logger.info(f'Head Importances = {head_importances}')
        torch.save(head_importances, os.path.join(args.output_dir, f'{mode}_{lang}_s{args.seed}_importances_100.pt'))
        torch.save(all_head_importances, os.path.join(args.output_dir, f'{mode}_{lang}_s{args.seed}_all_importances_100.pt'))
        return (None, None)
    if (nb_eval_steps == 0):
        results = {k: 0 for k in ['loss', 'precision', 'recall', 'f1']}
    else:
        eval_loss = (eval_loss / nb_eval_steps)
        preds = np.argmax(preds, axis=2)
        label_map = {i: label for (i, label) in enumerate(labels)}
        out_label_list = [[] for _ in range(out_label_ids.shape[0])]
        preds_list = [[] for _ in range(out_label_ids.shape[0])]
        for i in range(out_label_ids.shape[0]):
            for j in range(out_label_ids.shape[1]):
                if (out_label_ids[(i, j)] != pad_token_label_id):
                    out_label_list[i].append(label_map[out_label_ids[i][j]])
                    preds_list[i].append(label_map[preds[i][j]])
        results = {'loss': eval_loss, 'precision': precision_score(out_label_list, preds_list), 'recall': recall_score(out_label_list, preds_list), 'f1': f1_score(out_label_list, preds_list)}
    if print_result:
        logger.info(('***** Evaluation result %s in %s *****' % (prefix, lang)))
        for key in sorted(results.keys()):
            logger.info('  %s = %s', key, str(results[key]))
    return (results, preds_list)
def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix=, lang='en', lang2id=None, print_result=True, adapter_weight=None, lang_adapter_names=None, task_name=None, calc_weight_step=0):
    eval_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode=mode, lang=lang, lang2id=lang2id)
    args.eval_batch_size = (args.per_gpu_eval_batch_size * max(1, args.n_gpu))
    if args.get_attr:
        eval_sampler = (RandomSampler(eval_dataset) if (args.local_rank == (- 1)) else DistributedSampler(eval_dataset))
    else:
        eval_sampler = (SequentialSampler(eval_dataset) if (args.local_rank == (- 1)) else DistributedSampler(eval_dataset))
    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
    if (args.n_gpu > 1):
        model = torch.nn.DataParallel(model)
    logger.info(('***** Running evaluation %s in %s *****' % (prefix, lang)))
    logger.info('  Num examples = %d', len(eval_dataset))
    logger.info('  Batch size = %d', args.eval_batch_size)
    eval_loss = 0.0
    nb_eval_steps = 0
    preds = None
    out_label_ids = None
    model.eval()
    counter = 0
    head_importances = None
    all_head_importances = None
    for batch in tqdm(eval_dataloader, desc='Evaluating'):
        counter += 1
        logger.info(f'Batch number = {counter}')
        batch = tuple((t.to(args.device) for t in batch))
        if (calc_weight_step > 0):
            adapter_weight = calc_weight_multi(args, model, batch, lang_adapter_names, task_name, adapter_weight, calc_weight_step, lang=lang)
        if args.get_attr:
            inputs = {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3], 'adapter_weights': adapter_weight}
            if (args.model_type != 'distilbert'):
                inputs['token_type_ids'] = (batch[2] if (args.model_type in ['bert', 'xlnet']) else None)
            if (args.model_type == 'xlm'):
                inputs['langs'] = batch[4]
            inputs['output_attentions'] = True
            outputs = model(**inputs)
            (tmp_eval_loss, logits, attentions, kept_labels, kl_logits) = outputs
            attr_all = []
            res_attr = []
            input_len = int(inputs['attention_mask'][0].sum())
            example_head_importances = None
            logits = logits[0]
            for tar_layer in range(12):
                att = attentions[tar_layer][0]
                pred_labels = torch.argmax(logits, dim=(- 1))
                (scale_att, step) = scaled_input(att.data)
                scale_att.requires_grad_(True)
                attr_all = None
                prob_all = None
                for j_batch in range(1):
                    one_batch_att = scale_att[(j_batch * 16):((j_batch + 1) * 16)]
                    (_, grad) = model(input_ids=inputs['input_ids'], token_type_ids=inputs['token_type_ids'], attention_mask=inputs['attention_mask'], labels=inputs['labels'], tar_layer=tar_layer, tmp_score=one_batch_att, pred_labels=pred_labels)
                    grad = grad.sum(dim=0)
                    attr_all = (grad if (attr_all is None) else torch.add(attr_all, grad))
                attr_all = (attr_all[:, 0:input_len, 0:input_len] * step[:, 0:input_len, 0:input_len])
                if (example_head_importances is None):
                    example_head_importances = torch.amax(attr_all, dim=(1, 2)).unsqueeze(0)
                else:
                    tmp = torch.amax(attr_all, dim=(1, 2))
                    tmp = tmp.unsqueeze(0)
                    example_head_importances = torch.cat((example_head_importances, tmp), dim=0)
                res_attr.append(attr_all.data)
            all_head_importances = (example_head_importances.unsqueeze(0) if (all_head_importances is None) else torch.cat((all_head_importances, example_head_importances.unsqueeze(0)), dim=0))
            head_importances = (example_head_importances if (head_importances is None) else torch.add(head_importances, example_head_importances))
            if (counter == 100):
                break
            continue
        with torch.no_grad():
            inputs = {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3], 'adapter_weights': adapter_weight}
            if (args.model_type != 'distilbert'):
                inputs['token_type_ids'] = (batch[2] if (args.model_type in ['bert', 'xlnet']) else None)
            if (args.model_type == 'xlm'):
                inputs['langs'] = batch[4]
            outputs = model(**inputs)
            (tmp_eval_loss, logits) = outputs[:2]
            if (args.n_gpu > 1):
                tmp_eval_loss = tmp_eval_loss.mean()
            eval_loss += tmp_eval_loss.item()
        nb_eval_steps += 1
        if (preds is None):
            preds = logits.detach().cpu().numpy()
            out_label_ids = inputs['labels'].detach().cpu().numpy()
        else:
            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
            out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0)
    if args.get_attr:
        head_importances = (head_importances / counter)
        logger.info(f'Head Importances = {head_importances}')
        torch.save(head_importances, os.path.join(args.output_dir, f'{mode}_{lang}_s{args.seed}_importances_100.pt'))
        torch.save(all_head_importances, os.path.join(args.output_dir, f'{mode}_{lang}_s{args.seed}_all_importances_100.pt'))
        return (None, None)
    if (nb_eval_steps == 0):
        results = {k: 0 for k in ['loss', 'precision', 'recall', 'f1']}
    else:
        eval_loss = (eval_loss / nb_eval_steps)
        preds = np.argmax(preds, axis=2)
        label_map = {i: label for (i, label) in enumerate(labels)}
        out_label_list = [[] for _ in range(out_label_ids.shape[0])]
        preds_list = [[] for _ in range(out_label_ids.shape[0])]
        for i in range(out_label_ids.shape[0]):
            for j in range(out_label_ids.shape[1]):
                if (out_label_ids[(i, j)] != pad_token_label_id):
                    out_label_list[i].append(label_map[out_label_ids[i][j]])
                    preds_list[i].append(label_map[preds[i][j]])
        results = {'loss': eval_loss, 'precision': precision_score(out_label_list, preds_list), 'recall': recall_score(out_label_list, preds_list), 'f1': f1_score(out_label_list, preds_list)}
    if print_result:
        logger.info(('***** Evaluation result %s in %s *****' % (prefix, lang)))
        for key in sorted(results.keys()):
            logger.info('  %s = %s', key, str(results[key]))
    return (results, preds_list)