<commit_before># coding: utf-8
from __future__ import unicode_literals

from ...attrs import LEMMA, ORTH, PROB, IS_ALPHA
from ...parts_of_speech import NOUN, VERB

import pytest


@pytest.mark.parametrize('text1,text2', [
    ("Hello", "bye"), ("Hello", "hello"), ("Hello", "Hello,")])
def test_vocab_api_neq(en_vocab, text1, text2):
    assert en_vocab[text1].orth != en_vocab[text2].orth


@pytest.mark.parametrize('text', "Hello")
def test_vocab_api_eq(en_vocab, text):
    lex = en_vocab[text]
    assert en_vocab[text].orth == lex.orth


@pytest.mark.parametrize('text', ["example"])
def test_vocab_api_shape_attr(en_vocab, text):
    lex = en_vocab[text]
    assert lex.orth != lex.shape


@pytest.mark.parametrize('string,symbol', [
    ('IS_ALPHA', IS_ALPHA), ('NOUN', NOUN), ('VERB', VERB), ('LEMMA', LEMMA),
    ('ORTH', ORTH), ('PROB', PROB)])
def test_vocab_api_symbols(en_vocab, string, symbol):
    assert en_vocab.strings[string] == symbol


@pytest.mark.parametrize('text', "Hello")
def test_contains(en_vocab, text):
    _ = en_vocab[text]
    assert text in en_vocab
    assert "LKsdjvlsakdvlaksdvlkasjdvljasdlkfvm" not in en_vocab<commit_msg>Fix test name for consistency<commit_after># coding: utf-8
from __future__ import unicode_literals

from ...attrs import LEMMA, ORTH, PROB, IS_ALPHA
from ...parts_of_speech import NOUN, VERB

import pytest


@pytest.mark.parametrize('text1,text2', [
    ("Hello", "bye"), ("Hello", "hello"), ("Hello", "Hello,")])
def test_vocab_api_neq(en_vocab, text1, text2):
    assert en_vocab[text1].orth != en_vocab[text2].orth


@pytest.mark.parametrize('text', "Hello")
def test_vocab_api_eq(en_vocab, text):
    lex = en_vocab[text]
    assert en_vocab[text].orth == lex.orth


@pytest.mark.parametrize('text', ["example"])
def test_vocab_api_shape_attr(en_vocab, text):
    lex = en_vocab[text]
    assert lex.orth != lex.shape


@pytest.mark.parametrize('string,symbol', [
    ('IS_ALPHA', IS_ALPHA), ('NOUN', NOUN), ('VERB', VERB), ('LEMMA', LEMMA),
    ('ORTH', ORTH), ('PROB', PROB)])
def test_vocab_api_symbols(en_vocab, string, symbol):
    assert en_vocab.strings[string] == symbol


@pytest.mark.parametrize('text', "Hello")
def test_vocab_api_contains(en_vocab, text):
    _ = en_vocab[text]
    assert text in en_vocab
    assert "LKsdjvlsakdvlaksdvlkasjdvljasdlkfvm" not in en_vocab
{'fraction_non_alphanumeric': 0.1575809199318569, 'fraction_numerical': 0.00596252129471891, 'mean_word_length': 5.331536388140162, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3611483', 'n_tokens_mistral': 995, 'n_tokens_neox': 976, 'n_words': 183}