<commit_before>import warnings

question_words = set(["who", "what", "where", "when", "why", "how"])


def ends_with_question_word(dataset, idx):
    """Returns True if a question word is in the last three tokens of any sentence"""
    # HACK: For now (speedy POC), just use the BERT tokens
    # Eventually, we'd like to have access to the raw text (e.g., via the Spacy
    # tokenization) and have the two sentences separated for pair tasks

    # spacy_sentences = dataset.spacy_tokens[idx]
    # for spacy_sentence in spacy_sentences:
    #     if any(token.text in question_words for token in spacy_sentence):
    #         return True

    bert_ints = dataset.bert_tokens[idx]
    bert_tokens = dataset.bert_tokenizer.convert_ids_to_tokens(bert_ints)
    return any(token in question_words for token in bert_tokens[-3:])
    # match = any(token in question_words for token in bert_tokens[-3:])
    # if match:
    #     print(bert_tokens)
    # return match


def is_statement_has_question(dataset, idx):
    """Returns True if question word exists in statement that doesn't end with ?"""
    bert_ints = dataset.bert_tokens[idx]
    bert_tokens = dataset.bert_tokenizer.convert_ids_to_tokens(bert_ints)

    return (
        any(t.lower() in question_words for t in bert_tokens) and bert_tokens[-2] != "?"
    )


def ends_with_question_mark(dataset, idx):
    """Returns True if last token is '?" symbol"""
    bert_ints = dataset.bert_tokens[idx]
    bert_tokens = dataset.bert_tokenizer.convert_ids_to_tokens(bert_ints)

    # last token is '[SEP]'
    # check the second to last token for end of sentence
    return bert_tokens[-2] == "?"


# Functions which map a payload and index with an indicator if that example is in slice
# NOTE: the indexing is left to the functions so that extra fields helpful for slicing
# but not required by the model (e.g., spacy-based features) can be stored with the
# dataset but not necessarily passed back by __getitem__() which is called by the
# DataLoaders.
slice_functions = {
    "ends_with_question_word": ends_with_question_word,
    "ends_with_question_mark": ends_with_question_mark,
    "is_statement_has_question": is_statement_has_question,
}


def create_slice_labels(dataset, base_task_name, slice_name, verbose=False):
    """Returns a label set masked to include only those labels in the specified slice"""
    # TODO: break this out into more modular pieces one we have multiple slices
    slice_fn = slice_functions[slice_name]
    slice_indicators = [slice_fn(dataset, idx) for idx in range(len(dataset))]
    base_labels = dataset.labels[base_task_name]
    slice_labels = [
        label * indicator for label, indicator in zip(base_labels, slice_indicators)
    ]
    if verbose:
        print(f"Found {sum(slice_indicators)} examples in slice {slice_name}.")
        if not any(slice_labels):
            warnings.warn("No examples were found to belong to ")
<commit_msg>No longer create dictionary from function name to function<commit_after>import warnings

question_words = set(["who", "what", "where", "when", "why", "how"])


def ends_with_question_word(dataset, idx):
    """Returns True if a question word is in the last three tokens of any sentence"""
    # HACK: For now (speedy POC), just use the BERT tokens
    # Eventually, we'd like to have access to the raw text (e.g., via the Spacy
    # tokenization) and have the two sentences separated for pair tasks

    # spacy_sentences = dataset.spacy_tokens[idx]
    # for spacy_sentence in spacy_sentences:
    #     if any(token.text in question_words for token in spacy_sentence):
    #         return True

    bert_ints = dataset.bert_tokens[idx]
    bert_tokens = dataset.bert_tokenizer.convert_ids_to_tokens(bert_ints)
    return any(token in question_words for token in bert_tokens[-3:])
    # match = any(token in question_words for token in bert_tokens[-3:])
    # if match:
    #     print(bert_tokens)
    # return match


def is_statement_has_question(dataset, idx):
    """Returns True if question word exists in statement that doesn't end with ?"""
    bert_ints = dataset.bert_tokens[idx]
    bert_tokens = dataset.bert_tokenizer.convert_ids_to_tokens(bert_ints)

    return (
        any(t.lower() in question_words for t in bert_tokens) and bert_tokens[-2] != "?"
    )


def ends_with_question_mark(dataset, idx):
    """Returns True if last token is '?" symbol"""
    bert_ints = dataset.bert_tokens[idx]
    bert_tokens = dataset.bert_tokenizer.convert_ids_to_tokens(bert_ints)

    # last token is '[SEP]'
    # check the second to last token for end of sentence
    return bert_tokens[-2] == "?"


def dash_semicolon(dataset, idx):
    """Returns True if there is a dash or semicolon in sentence1"""
    bert_ints = dataset.bert_tokens[idx]
    bert_tokens = dataset.bert_tokenizer.convert_ids_to_tokens(bert_ints)
    return "-" in bert_tokens or ";" in bert_tokens


# Functions which map a payload and index with an indicator if that example is in slice
# NOTE: the indexing is left to the functions so that extra fields helpful for slicing
# but not required by the model (e.g., spacy-based features) can be stored with the
# dataset but not necessarily passed back by __getitem__() which is called by the
# DataLoaders.
# No longer needed: can do same thing as below with globals()[slice_name]
# slice_functions = {
#     "ends_with_question_word": ends_with_question_word,
#     "ends_with_question_mark": ends_with_question_mark,
#     "is_statement_has_question": is_statement_has_question,
# }


def create_slice_labels(dataset, base_task_name, slice_name, verbose=False):
    """Returns a label set masked to include only those labels in the specified slice"""
    # TODO: break this out into more modular pieces one we have multiple slices
    slice_fn = globals()[slice_name]
    slice_indicators = [slice_fn(dataset, idx) for idx in range(len(dataset))]
    base_labels = dataset.labels[base_task_name]
    slice_labels = [
        label * indicator for label, indicator in zip(base_labels, slice_indicators)
    ]
    if verbose:
        print(f"Found {sum(slice_indicators)} examples in slice {slice_name}.")
        if not any(slice_labels):
            warnings.warn("No examples were found to belong to ")

{'fraction_non_alphanumeric': 0.08667941363926067, 'fraction_numerical': 0.0014340344168260039, 'mean_word_length': 3.981746031746032, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5692818', 'n_tokens_mistral': 1920, 'n_tokens_neox': 1848, 'n_words': 637}