<commit_before>from __future__ import unicode_literals

from spacy.en import EN


def test_possess():
    tokens = EN.tokenize("Mike's")
    assert EN.lexicon.strings[tokens[0].sic] == "Mike"
    assert EN.lexicon.strings[tokens[1].sic] == "'s"
    assert len(tokens) == 2


def test_apostrophe():
    tokens = EN.tokenize("schools'")
    assert len(tokens) == 2
    assert tokens[1].string == "'"
    assert tokens[0].string == "schools"


def test_LL():
    tokens = EN.tokenize("we'll")
    assert len(tokens) == 2
    assert tokens[1].string == "will"
    assert tokens[0].string == "we"


def test_aint():
    tokens = EN.tokenize("ain't")
    assert len(tokens) == 2
    assert tokens[0].string == "are"
    assert tokens[1].string == "not"


def test_capitalized():
    tokens = EN.tokenize("can't")
    assert len(tokens) == 2
    tokens = EN.tokenize("Can't")
    assert len(tokens) == 2
    tokens = EN.tokenize("Ain't")
    assert len(tokens) == 2
    assert tokens[0].string == "Are"<commit_msg>* Add a couple more contractions tests<commit_after>from __future__ import unicode_literals

from spacy.en import EN


def test_possess():
    tokens = EN.tokenize("Mike's")
    assert EN.lexicon.strings[tokens[0].sic] == "Mike"
    assert EN.lexicon.strings[tokens[1].sic] == "'s"
    assert len(tokens) == 2


def test_apostrophe():
    tokens = EN.tokenize("schools'")
    assert len(tokens) == 2
    assert tokens[1].string == "'"
    assert tokens[0].string == "schools"


def test_LL():
    tokens = EN.tokenize("we'll")
    assert len(tokens) == 2
    assert tokens[1].string == "will"
    assert tokens[0].string == "we"


def test_aint():
    tokens = EN.tokenize("ain't")
    assert len(tokens) == 2
    assert tokens[0].string == "are"
    assert tokens[1].string == "not"


def test_capitalized():
    tokens = EN.tokenize("can't")
    assert len(tokens) == 2
    tokens = EN.tokenize("Can't")
    assert len(tokens) == 2
    tokens = EN.tokenize("Ain't")
    assert len(tokens) == 2
    assert tokens[0].string == "Are"


def test_punct():
    tokens = EN.tokenize("We've")
    assert len(tokens) == 2
    tokens = EN.tokenize("``We've")
    assert len(tokens) == 3
{'fraction_non_alphanumeric': 0.16750572082379864, 'fraction_numerical': 0.015560640732265447, 'mean_word_length': 3.416161616161616, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1432558', 'n_tokens_mistral': 790, 'n_tokens_neox': 733, 'n_words': 175}