<commit_before>import spacy
from ftlid import identify_language

from interfaces.SentenceOperation import SentenceOperation
from tasks.TaskTypes import TaskType


class CodeMixing(SentenceOperation):
    """This filter is used to identify code-mixed texts in a dataset.
    It checks that there is at least one sentence in the text where there
    are tokens representing at least `k` unique languages (with at least a
    `threshold` level of confidence that the token is of that language).
    """

    tasks = [
        TaskType.E2E_TASK,
        TaskType.TEXT_TAGGING,
        TaskType.TEXT_CLASSIFICATION,
    ]
    languages = [
        "en",
        "ru",
        "de",
        "fr",
        "it",
        "ja",
        "es",
        "ceb",
        "tr",
        "pt",
        "uk",
        "eo",
        "pl",
        "sv",
        "nl",
        "he",
        "zh",
        "hu",
        "ar",
        "ca",
        "fi",
        "cs",
        "fa",
        "sr",
        "el",<commit_msg>Add Keywords to Code-Mixing Filter<commit_after>import spacy
from ftlid import identify_language

from interfaces.SentenceOperation import SentenceOperation
from tasks.TaskTypes import TaskType


class CodeMixing(SentenceOperation):
    """This filter is used to identify code-mixed texts in a dataset.
    It checks that there is at least one sentence in the text where there
    are tokens representing at least `k` unique languages (with at least a
    `threshold` level of confidence that the token is of that language).
    """

    tasks = [
        TaskType.E2E_TASK,
        TaskType.TEXT_TAGGING,
        TaskType.TEXT_CLASSIFICATION,
    ]
    keywords = ['model-based', 'tokenizer-required']
    languages = [
        "en",
        "ru",
        "de",
        "fr",
        "it",
        "ja",
        "es",
        "ceb",
        "tr",
        "pt",
        "uk",
        "eo",
        "pl",
        "sv",
        "nl",
        "he",
        "zh",
        "hu",
        "ar",
        "ca",
        "fi",
        "cs",
        "fa",
        "sr",
        "el",
{'fraction_non_alphanumeric': 0.11073500967117988, 'fraction_numerical': 0.0009671179883945841, 'mean_word_length': 1.7586666666666666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '217401', 'n_tokens_mistral': 624, 'n_tokens_neox': 615, 'n_words': 194}