<jupyter_start><jupyter_text>HuggingFace Transformers for QA  Installing dependences<jupyter_code>#!python -m pip install --upgrade pip
#!pip install tensorflow
#!pip install transformers[tf-cpu]
#!pip install torch
#!pip install transformers[sentencepiece]<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>from transformers import pipeline
from transformers import AutoTokenizer, AutoModelForQuestionAnswering
import torch as pt<jupyter_output><empty_output><jupyter_text>Simple QA<jupyter_code>text = r"""
    🤗 Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides general-purpose
    architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet…) for Natural Language Understanding (NLU) and Natural
    Language Generation (NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between
    TensorFlow 2.0 and PyTorch.
    """
questions = [
    "How many pretrained models are available in 🤗 Transformers?",
    "How many languages are available in 🤗 Transformers?",
    "What does 🤗 Transformers provide?",
     "🤗 Transformers provides interoperability between which frameworks?",
]
qanlp = pipeline('question-answering')
result = qanlp(question=questions, context=text)
for res in result:
    print(f"Answer: '{res['answer']}', score: {round(res['score'], 4)}, start: {res['start']}, end: {res['end']}")<jupyter_output>Answer: 'over 32+', score: 0.5266, start: 268, end: 276
Answer: '100+', score: 0.5444, start: 298, end: 302
Answer: 'general-purpose
    architectures', score: 0.9541, start: 98, end: 131
Answer: 'TensorFlow 2.0 and PyTorch', score: 0.8424, start: 351, end: 377<jupyter_text>Selected model (example)<jupyter_code>MODEL = 'csarron/bert-base-uncased-squad-v1'
tokenizer = AutoTokenizer.from_pretrained(MODEL)
model = AutoModelForQuestionAnswering.from_pretrained(MODEL)
for question in questions:
    inputs = tokenizer(question, text, add_special_tokens=True, return_tensors="pt")
    input_ids = inputs["input_ids"].tolist()[0]

    text_tokens = tokenizer.convert_ids_to_tokens(input_ids)
    outputs = model(**inputs)
    answer_start_scores = outputs.start_logits
    answer_end_scores = outputs.end_logits
    
    answer_start = pt.argmax(answer_start_scores)  # Get the most likely beginning of answer with the argmax of the score
    answer_end = pt.argmax(answer_end_scores) + 1  # Get the most likely end of answer with the argmax of the score
    
    answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(input_ids[answer_start:answer_end]))
    start_predictions = pt.softmax(answer_start_scores, dim=1).tolist()[0]
    end_predictions = pt.softmax(answer_end_scores, dim=1).tolist()[0]

    print(f"Question: {question}")
    print(f"Answer: {answer}")
    print(f"Start Confidence: {start_predictions[answer_start]}")
    print(f"End Confidence: {end_predictions[answer_end]}")<jupyter_output>Question: How many pretrained models are available in 🤗 Transformers?
Answer: 32 +
Start Confidence: 0.6249579191207886
End Confidence: 0.0002813107566908002
Question: How many languages are available in 🤗 Transformers?
Answer: 100 +
Start Confidence: 0.8915950059890747
End Confidence: 0.17126747965812683
Question: What does 🤗 Transformers provide?
Answer: general - purpose architectures
Start Confidence: 0.9812468886375427
End Confidence: 7.347790233325213e-05
Question: 🤗 Transformers provides interoperability between which frameworks?
Answer: tensorflow 2. 0 and pytorch
Start Confidence: 0.9922741055488586
End Confidence: 0.09956756979227066<jupyter_text>QA https://huggingface.co/transformers/task_summary.htmlquestion-answering<jupyter_code># BERT

    # https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad

    #https://huggingface.co/csarron/bert-base-uncased-squad-v1
#MODEL = 'csarron/bert-base-uncased-squad-v1'

    #https://huggingface.co/phiyodr/bert-base-finetuned-squad2
#MODEL = 'phiyodr/bert-base-finetuned-squad2'

    #https://huggingface.co/phiyodr/bert-large-finetuned-squad2
#MODEL = 'phiyodr/bert-large-finetuned-squad2'

# ROBERTA

    #https://huggingface.co/csarron/roberta-base-squad-v1
#MODEL = 'csarron/roberta-base-squad-v1'

    #https://huggingface.co/phiyodr/roberta-large-finetuned-squad2
#MODEL = 'phiyodr/roberta-large-finetuned-squad2'

    #https://huggingface.co/deepset/roberta-base-squad2
#MODEL = 'deepset/roberta-base-squad2'
MODELS = [ 'phiyodr/roberta-large-finetuned-squad2', 'deepset/roberta-base-squad2',
          'phiyodr/bert-large-finetuned-squad2', 'phiyodr/bert-base-finetuned-squad2', 
          'valhalla/t5-base-squad',
          'tli8hf/unqover-bert-base-uncased-newsqa', 'tli8hf/unqover-roberta-base-newsqa',
          'tli8hf/unqover-bert-large-uncased-newsqa', 'tli8hf/unqover-roberta-large-newsqa']

#'bert-large-uncased-whole-word-masking-finetuned-squad',
#'csarron/roberta-base-squad-v1', 'csarron/bert-base-uncased-squad-v1',
MODEL = 'phiyodr/bert-base-finetuned-squad2'
tokenizer = AutoTokenizer.from_pretrained(MODEL)
model = AutoModelForQuestionAnswering.from_pretrained(MODEL)
# 'ConvBertForQuestionAnswering', 'LEDForQuestionAnswering', 'DistilBertForQuestionAnswering', 'AlbertForQuestionAnswering', 'CamembertForQuestionAnswering', 'BartForQuestionAnswering', 'MBartForQuestionAnswering', 'LongformerForQuestionAnswering', 'XLMRobertaForQuestionAnswering', 'SqueezeBertForQuestionAnswering', 
# 'FlaubertForQuestionAnsweringSimple', 'MobileBertForQuestionAnswering', 'XLMForQuestionAnsweringSimple', 'ElectraForQuestionAnswering', 'ReformerForQuestionAnswering', 'FunnelForQuestionAnswering', 'LxmertForQuestionAnswering', 'MPNetForQuestionAnswering', 'DebertaForQuestionAnswering'
# 'RobertaForQuestionAnswering', 'BertForQuestionAnswering', 'XLNetForQuestionAnsweringSimple', 
qa_specific = pipeline('question-answering', model=model, tokenizer=tokenizer)
beyonce_question = '<NAME> Beyonce attended which event together in August of 2011?'
amazon_question = 'Which name is also used to describe the Amazon rainforest in English?'
question_example = 'When was the Tower Theatre built?'
#response_example = '1939'
context_example = 'The popular neighborhood known as the Tower District is centered around the historic Tower Theatre, which is included on the National List of Historic Places. The theater was built in 1939 and is at Olive and Wishon Avenues in the heart of the Tower District. (The name of the theater refers to a well-known landmark water tower, which is actually in another nearby area). The Tower District neighborhood is just north of downtown Fresno proper, and one-half mile south of Fresno City College. Although the neighborhood was known as a residential area prior, the early commercial establishments of the Tower District began with small shops and services that flocked to the area shortly after World War II. The character of small local businesses largely remains today. To some extent, the businesses of the Tower District were developed due to the proximity of the original Fresno Normal School, (later renamed California State University at Fresno). In 1916 the college moved to what is now the site of Fresno City College one-half mile north of the Tower District.'
amazon_context_example= "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain 'Amazonas' in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
beyonce_context= 'In August, the couple attended the 2011 MTV Video Music Awards, at which Beyoncé performed "Love on Top" and started the performance saying "Tonight I want you to stand up on your feet, I want you to feel the love that\'s growing inside of me". At the end of the performance, she dropped her microphone, unbuttoned her blazer and rubbed her stomach, confirming her pregnancy she had alluded to earlier in the evening. Her appearance helped that year\'s MTV Video Music Awards become the most-watched broadcast in MTV history, pulling in 12.4 million viewers; the announcement was listed in Guinness World Records for "most tweets per second recorded for a single event" on Twitter, receiving 8,868 tweets per second and "Beyonce pregnant" was the most Googled term the week of August 29, 2011.'
print(qa_specific(question=question_example, context=context_example))
print(qa_specific(question=amazon_question, context=amazon_context_example))
print(qa_specific(question=beyonce_question, context=beyonce_context))
result = qa_specific(question=questions, context=text)
for res in result:
    print(f"Answer: '{res['answer']}', score: {round(res['score'], 4)}, start: {res['start']}, end: {res['end']}")
# XLNET - No funciona

#https://huggingface.co/jkgrad/xlnet-base-squadv2
#MODEL = 'jkgrad/xlnet-base-squadv2'

#MODEL = 'jkgrad/xlnet-base-cased-squad-quoref'

# https://huggingface.co/saburbutt/xlnet_large_tweetqa
#MODEL = 'saburbutt/xlnet_large_tweetqa'

#from transformers import XLNetTokenizerFast, XLNetForQuestionAnsweringSimple

#tokenizer = XLNetTokenizerFast.from_pretrained(MODEL)
#model = XLNetForQuestionAnsweringSimple.from_pretrained(MODEL)
qa_specific = pipeline('question-answering', model=model, tokenizer=tokenizer)
result = qa_specific(question=questions, context=text)
for res in result:
    print(f"Answer: '{res['answer']}', score: {round(res['score'], 4)}, start: {res['start']}, end: {res['end']}")
# T5
# https://huggingface.co/ozcangundes/T5-base-for-BioQA
MODEL = 'ozcangundes/T5-base-for-BioQA'
import torch
from transformers import T5ForConditionalGeneration, T5Tokenizer

tokenizer = T5Tokenizer.from_pretrained(MODEL)
model = T5ForConditionalGeneration.from_pretrained(MODEL)
def get_answer(question,context):
    source_encoding=tokenizer(
        question,
        context,
        max_length=512,
        padding="max_length",
        truncation="only_second",
        return_attention_mask=True,
        add_special_tokens=True,
        return_tensors="pt")
  
    generated_ids=model.generate(
          input_ids=source_encoding["input_ids"],
          attention_mask=source_encoding["attention_mask"])

    preds=[tokenizer.decode(gen_id, skip_special_tokens=True, clean_up_tokenization_spaces=True) for gen_id in generated_ids]

    return {'answer': "".join(preds)}
for query in questions:
    result = get_answer(query, text)
    print(result)
# https://huggingface.co/valhalla/t5-base-squad
MODEL = "valhalla/t5-base-squad"
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(MODEL)
model = AutoModelForSeq2SeqLM.from_pretrained(MODEL)
def get_answer(question, context):
    input_text = "question: %s  context: %s </s>" % (question, context)
    features = tokenizer([input_text], return_tensors='pt')
    
    out = model.generate(input_ids=features['input_ids'], attention_mask=features['attention_mask'])
    return tokenizer.decode(out[0], skip_special_tokens=True, clean_up_tokenization_spaces=True)
for query in questions:
    result = get_answer(query, text)
    print(result)<jupyter_output>32+
100+
general-purpose architectures
TensorFlow 2.0 and PyTorch<jupyter_text>Load dataset<jupyter_code>import pandas as pd
import json
MODELS = [ 'phiyodr/roberta-large-finetuned-squad2', 'deepset/roberta-base-squad2',
          'phiyodr/bert-large-finetuned-squad2', 'phiyodr/bert-base-finetuned-squad2', 
          'valhalla/t5-base-squad',
          'tli8hf/unqover-bert-base-uncased-newsqa', 'tli8hf/unqover-roberta-base-newsqa',
          'tli8hf/unqover-bert-large-uncased-newsqa', 'tli8hf/unqover-roberta-large-newsqa']

#'bert-large-uncased-whole-word-masking-finetuned-squad',
#'csarron/roberta-base-squad-v1', 'csarron/bert-base-uncased-squad-v1',
#SQUAD 2.0 DEV
#squad_dev = pd.read_excel('BERT-NER-POS-SQUAD-dev-queries.xlsx') #'NER-POS-SQUAD-dev-queries.xlsx')
squad_dev = pd.read_excel('BERT-NER-POS-SQUAD-train.xlsx') #'NER-POS-SQUAD-train-queries.xlsx')
#squad_dev = pd.read_excel('BERT-NER-POS-F-NEWSQA-ftunNEWSQA-ftunSQUAD.xlsx')
squad_dev.columns
#MODEL = #'phiyodr/bert-base-finetuned-squad2'
          ##'phiyodr/bert-large-finetuned-squad2' 
#squad_dev[MODEL] = None
sum(squad_dev[MODEL].isna())
#tokenizer = AutoTokenizer.from_pretrained(MODEL)
#model = AutoModelForQuestionAnswering.from_pretrained(MODEL)
# 'ConvBertForQuestionAnswering', 'LEDForQuestionAnswering', 'DistilBertForQuestionAnswering', 'AlbertForQuestionAnswering', 'CamembertForQuestionAnswering', 'BartForQuestionAnswering', 'MBartForQuestionAnswering', 'LongformerForQuestionAnswering', 'XLMRobertaForQuestionAnswering', 'SqueezeBertForQuestionAnswering', 
# 'FlaubertForQuestionAnsweringSimple', 'MobileBertForQuestionAnswering', 'XLMForQuestionAnsweringSimple', 'ElectraForQuestionAnswering', 'ReformerForQuestionAnswering', 'FunnelForQuestionAnswering', 'LxmertForQuestionAnswering', 'MPNetForQuestionAnswering', 'DebertaForQuestionAnswering'
# 'RobertaForQuestionAnswering', 'BertForQuestionAnswering', 'XLNetForQuestionAnsweringSimple', 
#qa_specific = pipeline('question-answering', model=model, tokenizer=tokenizer)
print(len(squad_dev.loc[squad_dev[MODEL]=='{}']))
# & (squad_dev['dataset'] == 'test')
context_restantes = squad_dev.loc[(squad_dev[MODEL].isna()), 'context']
print(len(context_restantes))
#context_restantes.values[0]
for context in context_restantes.unique():
    queries = squad_dev.loc[squad_dev['context'] == context, 'query'].values.tolist()
    queries = [element for element in queries if len(element) > 0 and str(element) != 'nan']
    if len(queries) > 0 and len(context) > 0:
        answers = qa_specific(question=queries, context=context)
        if len(answers) > 0:
            for query in queries:
                try:
                    squad_dev.loc[(squad_dev['context'] == context) & (squad_dev['query'] == query), 
                        MODEL] = str(answers[queries.index(query)])
                except:
                    squad_dev.loc[(squad_dev['context'] == context) & (squad_dev['query'] == query), 
                        MODEL] = '{}'
#squad_dev.to_excel('BERT-NER-POS-F-NEWSQA-ftunNEWSQA-ftunSQUAD-act.xlsx', index=False)
squad_dev.to_excel('BERT-NER-POS-SQUAD-train.xlsx', index=False)
# BERT-NER-POS-F-NEWSQA-ftunNEWSQA-ftunSQUAD.xlsx
#newsqa_fn.to_excel('BERT-NER-POS-F-NEWSQA-ftunNEWSQA-ftunSQUAD.xlsx')
print(sum(squad_dev[MODEL].isna()))
for context in context_restantes.unique():
    queries = squad_dev.loc[squad_dev['context'] == context, 'query'].values.tolist()
    for query in queries:
        answers = get_answer(query, context)
        squad_dev.loc[(squad_dev['context'] == context) & (squad_dev['query'] == query), MODEL] = str(answers)
#    print(sum(squad_dev[MODEL].isna()))
squad_dev.head()
squad_dev[MODEL].value_counts()<jupyter_output><empty_output><jupyter_text>Other pre-trained models for QA with TF<jupyter_code>from transformers import AutoTokenizer, TFAutoModelForQuestionAnswering, TFBertForMaskedLM, TFBertForQuestionAnswering
import tensorflow as tf
def get_pretrained_squad_model(model_name):
    model, tokenizer = None, None
    if model_name == "distilbertsquad1":
        tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased-distilled-squad",use_fast=True)
        model = TFBertForQuestionAnswering.from_pretrained("distilbert-base-cased-distilled-squad", from_pt=True)
    elif model_name == "distilbertsquad2": 
        tokenizer = AutoTokenizer.from_pretrained("twmkn9/distilbert-base-uncased-squad2",use_fast=True)
        model = TFAutoModelForQuestionAnswering.from_pretrained("twmkn9/distilbert-base-uncased-squad2", from_pt=True)
    elif model_name == "bertsquad2": 
        tokenizer = AutoTokenizer.from_pretrained("deepset/bert-base-cased-squad2",use_fast=True)
        model = TFBertForQuestionAnswering.from_pretrained("deepset/bert-base-cased-squad2", from_pt=True)
    elif model_name == "bertlargesquad2": 
        tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased",use_fast=True)
        model = TFBertForQuestionAnswering.from_pretrained("deepset/bert-large-uncased-whole-word-masking-squad2", from_pt=True)
    elif model_name == "albertbasesquad2": 
        tokenizer = AutoTokenizer.from_pretrained("twmkn9/albert-base-v2-squad2",use_fast=True)
        model = TFBertForQuestionAnswering.from_pretrained("twmkn9/albert-base-v2-squad2", from_pt=True)
    elif model_name == "distilrobertasquad2": 
        tokenizer = AutoTokenizer.from_pretrained("twmkn9/distilroberta-base-squad2",use_fast=True)
        model = TFBertForQuestionAnswering.from_pretrained("twmkn9/distilroberta-base-squad2", from_pt=True)
    elif model_name == "robertasquad2": 
        tokenizer = AutoTokenizer.from_pretrained("deepset/roberta-base-squad2",use_fast=True)
        model = TFAutoModelForQuestionAnswering.from_pretrained("deepset/roberta-base-squad2", from_pt=True)
    elif model_name == "bertlm":
        tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased",use_fast=True)
        model = TFBertForMaskedLM.from_pretrained("bert-base-uncased", from_pt=True)

    return model, tokenizer
# Choose from any set of HuggingFace models to use!
bqa_model, bqa_tokenizer = get_pretrained_squad_model("bertsquad2")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1092274559611243, 'fraction_numerical': 0.021370589209785188, 'mean_word_length': 5.672807663964628, 'pattern_counts': {'":': 8, '<': 28, '<?xml version=': 0, '>': 32, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '72783', 'license': 'MIT', 'n_tokens_mistral': 6401, 'n_tokens_neox': 5980, 'n_words': 1290, 'path': 'HuggingFace QA.ipynb', 'repo_name': 'laraolmos/tfm-nlp-qa', 'size': '18112'}