<commit_before>
import html
import re
import unicodedata

from bs4 import BeautifulSoup
import inflect
from pyparsing import Forward, Group, printables, OneOrMore, Suppress, Word, ZeroOrMore


def capwords(string):
    '''string.capwords with abbreviation handling'''
    return ' '.join(
        word.upper() if word.count('.') > 1 and not word.endswith("..")
        or word == "tv"
        else word.capitalize()
        for word in string.split()
    )


def check_answer(answer, response, inflect_engine = None):
    if not inflect_engine:
        inflect_engine = inflect.engine()

    # Unescape HTML entities in answer and extract text between HTML tags
    answer = BeautifulSoup(html.unescape(answer), "html.parser").get_text()
    # Replace in answer: \' -> '
    # Replace in response: ’ -> '
    # Replace: & -> and
    answer = answer.replace("\\'", "'").replace('&', "and")
    response = response.replace('’', "'").replace('&', "and")
    # Remove exclamation marks, periods, quotation marks, and interpuncts
    for character in '!."·':
        if character in answer:
            answer = answer.replace(character, "")
        if character in response:
            response = response.replace(character, "")
    # Remove diacritics
    answer = "".join(
        character for character in unicodedata.normalize("NFD", answer) 
        if not unicodedata.combining(character)
    )
    response = "".join(
        character for character in unicodedata.normalize("NFD", response) 
        if not unicodedata.combining(character)
    )
    # Remove extra whitespace
    # Make lowercase
    answer = ' '.join(answer.split()).lower()
    response = ' '.join(response.split()).lower()

    # Check removal of/replacement of - with space (prior to removing article prefixes)
    # Remove commas beforehand
    answer_copy = answer.replace(',', "")
    response_copy = response.replace(',', "")
    if answer_copy.replace('-', ' ') == response_copy.replace('-', ' '):
        return True
    if answer_copy.replace('-', "") == response_copy.replace('-', ""):
        return True

    # Remove article prefixes
    answer = remove_article_prefix(answer)
    response = remove_article_prefix(response)
    # Return False if empty response
    if not response:
        return False

    # Get items in lists
    answer_items = [item.strip() for item in answer.split(',')]
    answer_items[-1:] = [
        item.strip() for item in answer_items[-1].split("and") if item
    ]
    response_items = [item.strip() for item in response.split(',')]
    response_items[-1:] = [
        item.strip() for item in response_items[-1].split("and") if item
    ]
    # Return False if only "and"
    if not response_items:
        return False
    # Remove article prefixes
    for index, item in enumerate(answer_items):
        answer_items[index] = remove_article_prefix(item)
    for index, item in enumerate(response_items):<commit_msg>[Units] Handle empty answers after processing in trivia.check_answer<commit_after>
import html
import re
import unicodedata

from bs4 import BeautifulSoup
import inflect
from pyparsing import Forward, Group, printables, OneOrMore, Suppress, Word, ZeroOrMore


def capwords(string):
    '''string.capwords with abbreviation handling'''
    return ' '.join(
        word.upper() if word.count('.') > 1 and not word.endswith("..")
        or word == "tv"
        else word.capitalize()
        for word in string.split()
    )


def check_answer(answer, response, inflect_engine = None):
    if not inflect_engine:
        inflect_engine = inflect.engine()

    # Unescape HTML entities in answer and extract text between HTML tags
    answer = BeautifulSoup(html.unescape(answer), "html.parser").get_text()
    # Replace in answer: \' -> '
    # Replace in response: ’ -> '
    # Replace: & -> and
    answer = answer.replace("\\'", "'").replace('&', "and")
    response = response.replace('’', "'").replace('&', "and")
    # Remove exclamation marks, periods, quotation marks, and interpuncts
    for character in '!."·':
        if character in answer:
            answer = answer.replace(character, "")
        if character in response:
            response = response.replace(character, "")
    # Remove diacritics
    answer = "".join(
        character for character in unicodedata.normalize("NFD", answer) 
        if not unicodedata.combining(character)
    )
    response = "".join(
        character for character in unicodedata.normalize("NFD", response) 
        if not unicodedata.combining(character)
    )
    # Remove extra whitespace
    # Make lowercase
    answer = ' '.join(answer.split()).lower()
    response = ' '.join(response.split()).lower()

    # Check removal of/replacement of - with space (prior to removing article prefixes)
    # Remove commas beforehand
    answer_copy = answer.replace(',', "")
    response_copy = response.replace(',', "")
    if answer_copy.replace('-', ' ') == response_copy.replace('-', ' '):
        return True
    if answer_copy.replace('-', "") == response_copy.replace('-', ""):
        return True

    # Remove article prefixes
    answer = remove_article_prefix(answer)
    response = remove_article_prefix(response)
    # Return False if empty response or answer
    if not response or not answer:
        return False

    # Get items in lists
    answer_items = [item.strip() for item in answer.split(',')]
    answer_items[-1:] = [
        item.strip() for item in answer_items[-1].split("and") if item
    ]
    response_items = [item.strip() for item in response.split(',')]
    response_items[-1:] = [
        item.strip() for item in response_items[-1].split("and") if item
    ]
    # Return False if only "and"
    if not response_items:
        return False
    # Remove article prefixes
    for index, item in enumerate(answer_items):
        answer_items[index] = remove_article_prefix(item)
    for index, item in enumerate(response_items):
{'fraction_non_alphanumeric': 0.12682681001175877, 'fraction_numerical': 0.0020157903578027886, 'mean_word_length': 3.012129380053908, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '457442', 'n_tokens_mistral': 1735, 'n_tokens_neox': 1683, 'n_words': 503}