<jupyter_start><jupyter_text>Multilingual QA searchDownloads and parses several coronavirus related WHO FAQ, uses Multilingual Universal Sentence Encoder QA, to encode paragraphs or list elementsUSE supports 16 languages (Arabic, Chinese-simplified, Chinese-traditional, English, French, German, Italian, Japanese, Korean, Dutch, Polish, Portuguese, Spanish, Thai, Turkish, Russian)uses some code from https://github.com/wearetriple/ai-faqbot-who/blob/master/Corona_WHO_FAQ.ipynb<jupyter_code>import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_text
import re

import requests
from bs4 import BeautifulSoup

import numpy as np
import pandas as pd

pd.set_option("max_colwidth", 200)<jupyter_output><empty_output><jupyter_text>Download and parse QA sites<jupyter_code>def iter_who_faq(url="https://www.who.int/news-room/q-a-detail/q-a-coronaviruses"):
    page = requests.get(url)
    soup = BeautifulSoup(page.text, "lxml")
    for panel in soup.find_all(attrs={"class": "sf-accordion__panel"}):
        question = panel.find_all("a")[0].text.strip()
        answers = [p.text.strip() for p in panel.find_all("p") if p.text.strip()] + [
            list_element.text.strip() for list_element in panel.find_all("li")
        ]
        if not answers:
            print(url, question)
        for answer in answers:
            yield question, answer, url
urls = [
    "https://www.who.int/news-room/q-a-detail/q-a-coronaviruses",
    "https://www.who.int/news-room/q-a-detail/q-a-on-covid-19-pregnancy-childbirth-and-breastfeeding",
    "https://www.who.int/news-room/q-a-detail/q-a-on-covid-19-hiv-and-antiretrovirals",
    "https://www.who.int/news-room/q-a-detail/q-a-similarities-and-differences-covid-19-and-influenza",
    "https://www.who.int/news-room/q-a-detail/q-a-on-mass-gatherings-and-covid-19",
    "https://www.who.int/news-room/q-a-detail/q-a-on-smoking-and-covid-19",
    "https://www.who.int/news-room/q-a-detail/be-active-during-covid-19",
    "https://www.who.int/news-room/q-a-detail/malaria-and-the-covid-19-pandemic",
    "https://www.who.int/news-room/q-a-detail/q-a-on-infection-prevention-and-control-for-health-care-workers-caring-for-patients-with-suspected-or-confirmed-2019-ncov",
    "https://www.who.int/news-room/q-a-detail/middle-east-respiratory-syndrome-coronavirus-(mers-cov)",
]


def iter_who_faqs(urls=urls):
    for url in urls:
        for out in iter_who_faq(url=url):
            yield out
%%time
data = pd.DataFrame(data=list(iter_who_faqs()), columns=["Context", "Answer", "Source"])
data.head()<jupyter_output>CPU times: user 592 ms, sys: 13.2 ms, total: 605 ms
Wall time: 2.72 s<jupyter_text>Preprocess Sentencesbecause USE has never seen covid or covid-19 but has seen coronavirus<jupyter_code>%%time
def preprocess_sentences(input_sentences):
    return [
        re.sub(r"(covid-19|covid)", "coronavirus", input_sentence, flags=re.I)
        for input_sentence in input_sentences
    ]


# Load module containing USE
module = hub.load(
    "https://tfhub.dev/google/universal-sentence-encoder-multilingual-qa/3"
)
%%time
# Create response embeddings
response_encodings = module.signatures["response_encoder"](
    input=tf.constant(preprocess_sentences(data.Answer)),
    context=tf.constant(preprocess_sentences(data.Context)),
)["outputs"]<jupyter_output>CPU times: user 44.2 s, sys: 8.15 s, total: 52.3 s
Wall time: 15.2 s<jupyter_text>And test some new questions...<jupyter_code>%%time
test_questions = [
    "What about pregnant women?",
    "Wat is de lengte van de incubatietijd?",
    "Are animals contagious COVID-19?",
    "Are there medicine against the coronavirus?",
    "Can I breastfead when I have COVID-19?",
    "Should I stay inside the house?",  # English questions are also possible.
    "Kann ich mit meinem Hund spazieren gehen?",  # As well as German, and all the other languages supported by use-multilingual.
]

# Create encodings for test questions
question_encodings = module.signatures["question_encoder"](
    tf.constant(preprocess_sentences(test_questions))
)["outputs"]

# Get the responses
test_responses = data.loc[
    np.argmax(np.inner(question_encodings, response_encodings), axis=1),
    ["Answer", "Source"],
]

# Show them in a dataframe
pd.DataFrame(
    {
        "Test Questions": test_questions,
        "Test Responses": test_responses["Answer"],
        "Test Sources": test_responses["Source"],
    }
)
from timeit import default_timer as timer
question = input()
start = timer()
question_encoding = module.signatures["question_encoder"](
    tf.constant(preprocess_sentences([question]))
)["outputs"]
test_response = data.loc[
    np.argmax(np.inner(question_encoding, response_encodings), axis=1),
    ["Answer", "Source"],
]
end = timer()
print(test_response["Answer"].values[0])
print("This response was gathered from:")
print(test_response["Source"].values[0])
print("This query took {} seconds".format(end - start))<jupyter_output>Does smoking make it easier to get COVID19?
{'fraction_non_alphanumeric': 0.13905562224889956, 'fraction_numerical': 0.012805122048819529, 'mean_word_length': 4.3237486687965925, 'pattern_counts': {'":': 4, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 13, 'lorem ipsum': 0, 'www.': 11, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '113082', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1783, 'n_tokens_neox': 1686, 'n_words': 351, 'path': 'notebooks/WHO_question_answering/1.0-mjt-USE-qa.ipynb', 'repo_name': 'MarvinT/covid_analysis', 'size': '4998'}