<jupyter_start><jupyter_text>Main<jupyter_code>bert_tokenizer = AutoTokenizer.from_pretrained("/content/drive/My Drive/nlp_dataset/bert-base-uncased")
bert_model = AutoModelForQuestionAnswering.from_pretrained("/content/drive/My Drive/nlp_dataset/bert-base-uncased")
distilbert_tokenizer = AutoTokenizer.from_pretrained("/content/drive/My Drive/nlp_dataset/distilbert-base-uncased")
distilbert_model = AutoModelForQuestionAnswering.from_pretrained("/content/drive/My Drive/nlp_dataset/distilbert-base-uncased")
# this processor loads the SQuAD2.0 dev set examples
processor = SquadV2Processor()
dev_examples = processor.get_dev_examples("/content/squad", filename="dev-v2.0.json")
print(len(dev_examples))
# generate some maps to help us identify examples of interest
qid_to_example_index = {example.qas_id: i for i, example in enumerate(dev_examples)}
qid_to_has_answer = {example.qas_id: bool(example.answers) for example in dev_examples}
answer_qids = [qas_id for qas_id, has_answer in qid_to_has_answer.items() if has_answer]
no_answer_qids = [qas_id for qas_id, has_answer in qid_to_has_answer.items() if not has_answer]<jupyter_output><empty_output><jupyter_text>BERT TESTING<jupyter_code>dev_set_testing("/content/drive/My Drive/nlp_dataset/bert-base-uncased/", 
                bert_model, bert_tokenizer, dev_examples, answer_qids, no_answer_qids)
evaluate_model("/content/drive/My Drive/nlp_dataset/bert-base-uncased/", 
                bert_model, bert_tokenizer, dev_examples, answer_qids, no_answer_qids)<jupyter_output>METRICS FOR DEV SET WITHOUT POSTPROCESSING
OrderedDict([('exact', 69.34220500294786),
             ('f1', 73.80082311322276),
             ('total', 11873),
             ('HasAns_exact', 60.239541160593795),
             ('HasAns_f1', 69.16956356668244),
             ('HasAns_total', 5928),
             ('NoAns_exact', 78.4188393608074),
             ('NoAns_f1', 78.4188393608074),
             ('NoAns_total', 5945),
             ('best_exact', 69.52749936831466),
             ('best_exact_thresh', -6.801867485046387),
             ('best_f1', 73.8008231132222),
             ('best_f1_thresh', -3.7661972045898438)])
METRICS FOR DEV SET WITH POSTPROCESSING
OrderedDict([('exact', 69.43485218563126),
             ('f1', 73.8232516483557),
             ('total', 11873),
             ('HasAns_exact', 60.35762483130904),
             ('HasAns_f1', 69.14700857303089),
             ('HasAns_total', 5928),
             ('NoAns_exact', 78.4861227922624),
             ('NoAns_f1', 78.486122792262[...]<jupyter_text>DISTILBERT TESTING<jupyter_code>dev_set_testing("/content/drive/My Drive/nlp_dataset/distilbert-base-uncased/", 
                distilbert_model, distilbert_tokenizer, dev_examples, answer_qids, no_answer_qids)
evaluate_model("/content/drive/My Drive/nlp_dataset/distilbert-base-uncased/", 
                bert_model, bert_tokenizer, dev_examples, answer_qids, no_answer_qids)<jupyter_output>METRICS FOR DEV SET WITHOUT POSTPROCESSING
OrderedDict([('exact', 64.57508633032931),
             ('f1', 68.54706579411912),
             ('total', 11873),
             ('HasAns_exact', 53.9136302294197),
             ('HasAns_f1', 61.86897978636555),
             ('HasAns_total', 5928),
             ('NoAns_exact', 75.20605550883096),
             ('NoAns_f1', 75.20605550883096),
             ('NoAns_total', 5945),
             ('best_exact', 64.71826833993093),
             ('best_exact_thresh', -5.177046895027161),
             ('best_f1', 68.54706579411919),
             ('best_f1_thresh', -3.7676496505737305)])
METRICS FOR DEV SET WITH POSTPROCESSING
OrderedDict([('exact', 64.62562115724754),
             ('f1', 68.53062384308238),
             ('total', 11873),
             ('HasAns_exact', 53.93049932523617),
             ('HasAns_f1', 61.75170325386568),
             ('HasAns_total', 5928),
             ('NoAns_exact', 75.2901597981497),
             ('NoAns_f1', 75.2901597981[...]<jupyter_text>TESTING ON AUGMENTED DATASET<jupyter_code># this load augmented data
processor = SquadV2Processor()
dev_examples = processor.get_dev_examples("/content/drive/My Drive/nlp_dataset/", filename="augmented_dev.json")
print(len(dev_examples))
# generate some maps to help us identify examples of interest
qid_to_example_index = {example.qas_id: i for i, example in enumerate(dev_examples)}
qid_to_has_answer = {example.qas_id: bool(example.answers) for example in dev_examples}
answer_qids = [qas_id for qas_id, has_answer in qid_to_has_answer.items() if has_answer]
no_answer_qids = [qas_id for qas_id, has_answer in qid_to_has_answer.items() if not has_answer]
dev_set_testing("/content/drive/My Drive/nlp_dataset/bert-base-uncased/", 
                bert_model, bert_tokenizer, dev_examples, answer_qids, no_answer_qids)
evaluate_model("/content/drive/My Drive/nlp_dataset/bert-base-uncased/", 
                bert_model, bert_tokenizer, dev_examples, answer_qids, no_answer_qids)
dev_set_testing("/content/drive/My Drive/nlp_dataset/distilbert-base-uncased/", 
                distilbert_model, distilbert_tokenizer, dev_examples, answer_qids, no_answer_qids)
evaluate_model("/content/drive/My Drive/nlp_dataset/distilbert-base-uncased/", 
                bert_model, bert_tokenizer, dev_examples, answer_qids, no_answer_qids)<jupyter_output>METRICS FOR DEV SET WITHOUT POSTPROCESSING
OrderedDict([('exact', 51.06544260085909),
             ('f1', 55.48120675166491),
             ('total', 11873),
             ('HasAns_exact', 15.24966261808367),
             ('HasAns_f1', 24.093854210950795),
             ('HasAns_total', 5928),
             ('NoAns_exact', 86.77880571909168),
             ('NoAns_f1', 86.77880571909168),
             ('NoAns_total', 5945),
             ('best_exact', 52.08456161037648),
             ('best_exact_thresh', -8.188229084014893),
             ('best_f1', 55.455939338206),
             ('best_f1_thresh', -3.447478547692299)])
METRICS FOR DEV SET WITH POSTPROCESSING
OrderedDict([('exact', 51.579213341194304),
             ('f1', 55.44090730463834),
             ('total', 11873),
             ('HasAns_exact', 14.018218623481781),
             ('HasAns_f1', 21.752680908901812),
             ('HasAns_total', 5928),
             ('NoAns_exact', 89.03280067283431),
             ('NoAns_f1', 89.0328006[...]
{'fraction_non_alphanumeric': 0.12387137652463172, 'fraction_numerical': 0.13448439727546332, 'mean_word_length': 3.440225035161744, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '45378', 'license': 'MIT', 'n_tokens_mistral': 2826, 'n_tokens_neox': 2237, 'n_words': 343, 'path': 'Optimizing QA models/qa_testing.ipynb', 'repo_name': 'ayush-bisht/NLP', 'size': '6313'}