<commit_before># coding: utf-8
from __future__ import unicode_literals

from ...language import Language
from ...attrs import LANG
from ...fr.language_data import get_tokenizer_exceptions, STOP_WORDS
from ...language_data.punctuation import TOKENIZER_INFIXES, ALPHA

import pytest


@pytest.fixture
def fr_tokenizer_w_infix():
    SPLIT_INFIX = r'(?<=[{a}]\')(?=[{a}])'.format(a=ALPHA)

    # create new Language subclass to add to default infixes
    class French(Language):
        lang = 'fr'

        class Defaults(Language.Defaults):
            lex_attr_getters = dict(Language.Defaults.lex_attr_getters)
            lex_attr_getters[LANG] = lambda text: 'fr'
            tokenizer_exceptions = get_tokenizer_exceptions()
            stop_words = STOP_WORDS
            infixes = TOKENIZER_INFIXES + [SPLIT_INFIX]

    return French.Defaults.create_tokenizer()


@pytest.mark.parametrize('text,expected_tokens', [("l'avion", ["l'", "avion"]),
                                                  ("j'ai", ["j'", "ai"])])
def test_issue768(fr_tokenizer_w_infix, text, expected_tokens):
    """Allow zero-width 'infix' token during the tokenization process."""
    tokens = fr_tokenizer_w_infix(text)
    assert len(tokens) == 2
    assert [t.text for t in tokens] == expected_tokens<commit_msg>Fix import and tokenizer exceptions<commit_after># coding: utf-8
from __future__ import unicode_literals

from ...language import Language
from ...attrs import LANG
from ...fr.stop_words import STOP_WORDS
from ...fr.tokenizer_exceptions import TOKENIZER_EXCEPTIONS
from ...language_data.punctuation import TOKENIZER_INFIXES, ALPHA
from ...util import update_exc

import pytest


@pytest.fixture
def fr_tokenizer_w_infix():
    SPLIT_INFIX = r'(?<=[{a}]\')(?=[{a}])'.format(a=ALPHA)

    # create new Language subclass to add to default infixes
    class French(Language):
        lang = 'fr'

        class Defaults(Language.Defaults):
            lex_attr_getters = dict(Language.Defaults.lex_attr_getters)
            lex_attr_getters[LANG] = lambda text: 'fr'
            tokenizer_exceptions = update_exc(TOKENIZER_EXCEPTIONS)
            stop_words = STOP_WORDS
            infixes = TOKENIZER_INFIXES + [SPLIT_INFIX]

    return French.Defaults.create_tokenizer()


@pytest.mark.parametrize('text,expected_tokens', [("l'avion", ["l'", "avion"]),
                                                  ("j'ai", ["j'", "ai"])])
def test_issue768(fr_tokenizer_w_infix, text, expected_tokens):
    """Allow zero-width 'infix' token during the tokenization process."""
    tokens = fr_tokenizer_w_infix(text)
    assert len(tokens) == 2
    assert [t.text for t in tokens] == expected_tokens
{'fraction_non_alphanumeric': 0.11698956780923994, 'fraction_numerical': 0.0037257824143070045, 'mean_word_length': 3.3517017828200975, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '117443', 'n_tokens_mistral': 890, 'n_tokens_neox': 855, 'n_words': 196}