<jupyter_start><jupyter_text>ensembling model for prediction<jupyter_code>from transformers import AutoModelForQuestionAnswering,AutoTokenizer
from transformers import squad_convert_examples_to_features
from transformers.data.processors.squad import SquadV2Processor
import torch
import torch.nn.functional as F
import time
from tqdm import tqdm
import numpy as np
squad_data_dir='./SQUAD_V2'
# dev-v2.0.json
processor=SquadV2Processor()
eval_examples=processor.get_dev_examples(squad_data_dir,filename='dev-v2.0.json')
train_examples=processor.get_train_examples(squad_data_dir,filename='train-v2.0.json')
for i,example in enumerate(train_examples):
    print(example.qas_id,example.question_text)
    print(example.answer_text)
    print(example.context_text)
    if i==2:break
model_dirs=['./mrm8488/bert-medium-finetuned-squadv2','./mrm8488/bert-small-finetuned-squadv2','./mrm8488/bert-mini-5-finetuned-squadv2']
pretrained_model_path1='/home/stu_18701958249/hub_dir/2020-ai-for-nlp-summer/All_tasks_solved_by_transformers/Extractive_Question_Answering/pretrained_model'
models=[AutoModelForQuestionAnswering.from_pretrained(model_dir,return_dict=True) for model_dir in model_dirs]
tokenizers=[AutoTokenizer.from_pretrained(model_dir) for model_dir in model_dirs]
def get_ensembled_result(many_start_probs,many_end_probs,mode='equal'):
    model_nums=len(many_start_probs)
    if mode=='equal':
        ranks=[1.0/model_nums]*model_nums
    elif mode=='ranks':
        ranks=[index/(model_nums*(model_nums+1)*0.5) for index in range(1,model_nums+1)]
    # get the average combined prob from several models
    avg_start_probs,avg_end_probs=0,0
    reference_combined_prob=0.0
    for i in range(model_nums):
        avg_start_probs+=many_start_probs[i]*ranks[i]
        avg_end_probs+=many_end_probs[i]*ranks[i]
        
        combined_matrix=np.matmul(many_start_probs[i][:,np.newaxis],many_end_probs[i][np.newaxis,:])*ranks[i]**2
        max_combined_prob=np.max(combined_matrix)
        if max_combined_prob>reference_combined_prob:
            reference_combined_prob=max_combined_prob
            start,end=np.argwhere(combined_matrix==max_combined_prob)[0]
            model_index,max_answer_start,max_answer_end,max_combined_prob=i,start,end,max_combined_prob       
    # get the average combined prob from several models
    total_combined_avg_result=np.matmul(avg_start_probs[:,np.newaxis],avg_end_probs[np.newaxis,:])
    avg_combined_prob=np.max(total_combined_avg_result)
    answer_start,answer_end=np.argwhere(total_combined_avg_result==avg_combined_prob)[0]
    return model_index,answer_start,answer_end
tokenizers[0].tokenize('I love you')
def get_ensembled_answer(models,tokenizers,question,context,max_sequence_len=384,verbose=False,device='cpu'):
    all_inputs=[tokenizer.encode_plus(question, context, max_length=max_sequence_len, truncation=True, padding='max_length', return_tensors='pt') for tokenizer in tokenizers]
    all_start_probs,all_end_probs=[],[]
    with torch.no_grad():
        for index,model in enumerate(models):
            all_inputs[index].to(device)
            result=model(**all_inputs[index])
            all_start_probs.append(torch.nn.functional.softmax(result.start_logits,dim=1).detach().cpu().numpy()[0])
            all_end_probs.append(torch.nn.functional.softmax(result.end_logits,dim=1).detach().cpu().numpy()[0])
    model_index,answer_start,answer_end=get_ensembled_result(all_start_probs,all_end_probs,mode='equal')
    answer=tokenizers[model_index].decode(all_inputs[model_index]['input_ids'][0,answer_start:answer_end+1],skip_special_tokens=True)
    if verbose:
        print('question:',question)
        print('context:',context)
        print('answer:',answer)
    return answer
predictions={}
trues={}
start_time=time.time()
total_examples=0
correct_examples=0
device='cuda:0' if torch.cuda.is_available() else 'cpu'
for model in models:
    model.to(device)
    model.eval()
for i,example in enumerate(tqdm(train_examples[:10])):
    one_example_start_time=time.time()
    answer=get_ensembled_answer(models,tokenizers,example.question_text,example.context_text,max_sequence_len=384,verbose=False,device=device)
    predictions[example.qas_id]=answer
    trues[example.qas_id]=example.answer_text.lower()
    correct_examples+=int(answer==example.answer_text.lower() or answer in example.answer_text.lower() or example.answer_text.lower() in answer)
    total_examples+=1
    print('it takes {} to predict one example.'.format(time.time()-one_example_start_time))
print('it takes {} to predict 10 examples.'.format(time.time()-start_time))
print(predictions)
print('correct rate:',correct_examples/total_examples)
print(trues)<jupyter_output>10%|█         | 1/10 [00:17<02:41, 17.96s/it]
{'fraction_non_alphanumeric': 0.1190176322418136, 'fraction_numerical': 0.019731318219983206, 'mean_word_length': 6.399068322981367, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '576106', 'license': 'MIT', 'n_tokens_mistral': 1721, 'n_tokens_neox': 1665, 'n_words': 194, 'path': 'ensemble_technology_for_question_answering_model/Squadv2_Ensemble_Inference.ipynb', 'repo_name': 'huakeda1/Basic-algorithm-and-framework-study-for-AI', 'size': '4764'}