<commit_before>from textwrap import dedent

import pytest

from kinko.errors import Errors
from kinko.tokenizer import tokenize as _tokenize
from kinko.tokenizer import Token, TokenizerError


NL = (Token.NEWLINE, '\n')

EOF = (Token.EOF, '')


def tokenize(src, errors=None):
    src = dedent(src).strip()
    return _tokenize(src, errors)


def check_tokens(string, tokens):
    left = [(kind, value) for (kind, value, loc) in tokenize(string)]
    assert left == tokens


def check_errors(string, messages):
    errors = Errors()
    try:
        list(tokenize(string, errors))
    except TokenizerError:
        errors_map = {e.message for e in errors.list}
        assert errors_map == set(messages)
    else:
        raise AssertionError('No errors')


def test_symbol():
    check_tokens(
        'a foo1 foo.bar foo-bar foo/bar ./foo foo_bar Foo',
        [
            (Token.SYMBOL, 'a'),
            (Token.SYMBOL, 'foo1'),
            (Token.SYMBOL, 'foo.bar'),
            (Token.SYMBOL, 'foo-bar'),
            (Token.SYMBOL, 'foo/bar'),
            (Token.SYMBOL, './foo'),
            (Token.SYMBOL, 'foo_bar'),
            (Token.SYMBOL, 'Foo'), NL,
            EOF,
        ],
    )


def test_string():
    check_tokens(
        '"foo" "foo.bar" "foo-bar" "foo/bar" "foo_bar" "Foo" '
        '":foo" "#foo" "{}" "[]" "()" "123"',
        [
            (Token.STRING, 'foo'),
            (Token.STRING, 'foo.bar'),
            (Token.STRING, 'foo-bar'),
            (Token.STRING, 'foo/bar'),
            (Token.STRING, 'foo_bar'),
            (Token.STRING, 'Foo'),
            (Token.STRING, ':foo'),
            (Token.STRING, '#foo'),
            (Token.STRING, '{}'),
            (Token.STRING, '[]'),
            (Token.STRING, '()'),
            (Token.STRING, '123'), NL,
            EOF,
        ],
    )


def test_incomplete_string():
    check_errors(
        '"foo',
        ['String does not and at EOF'],
    )


def test_number():
    check_tokens(
        '1 2.3 4.5d 6d.7 8d 9...',
        [
            (Token.NUMBER, '1'),
            (Token.NUMBER, '2.3'),
            (Token.NUMBER, '4.5d'),
            (Token.NUMBER, '6d.7'),
            (Token.NUMBER, '8d'),
            (Token.NUMBER, '9...'), NL,
            EOF,
        ],
    )


def test_keyword():
    check_tokens(
        ':foo :foo-bar :foo_bar',
        [
            (Token.KEYWORD, 'foo'),
            (Token.KEYWORD, 'foo-bar'),
            (Token.KEYWORD, 'foo_bar'), NL,
            EOF,
        ],
    )


def test_laceholder():
    check_tokens(
        '#foo #foo-bar #foo_bar #foo.bar',
        [
            (Token.PLACEHOLDER, 'foo'),
            (Token.PLACEHOLDER, 'foo-bar'),
            (Token.PLACEHOLDER, 'foo_bar'),
            (Token.PLACEHOLDER, 'foo.bar'), NL,
            EOF,
        ],
    )


def test_brackets():
    check_tokens(<commit_msg>#3 Covered in tests all tokenizer errors<commit_after>from textwrap import dedent

import pytest

from kinko.errors import Errors
from kinko.tokenizer import tokenize as _tokenize
from kinko.tokenizer import Token, TokenizerError


NL = (Token.NEWLINE, '\n')

EOF = (Token.EOF, '')


def tokenize(src, errors=None):
    src = dedent(src).strip()
    return _tokenize(src, errors)


def check_tokens(string, tokens):
    left = [(kind, value) for (kind, value, loc) in tokenize(string)]
    assert left == tokens


def check_errors(string, messages):
    errors = Errors()
    try:
        list(tokenize(string, errors))
    except TokenizerError:
        errors_map = {e.message for e in errors.list}
        assert errors_map == set(messages)
    else:
        raise AssertionError('No errors')


def test_symbol():
    check_tokens(
        'a foo1 foo.bar foo-bar foo/bar ./foo foo_bar Foo',
        [
            (Token.SYMBOL, 'a'),
            (Token.SYMBOL, 'foo1'),
            (Token.SYMBOL, 'foo.bar'),
            (Token.SYMBOL, 'foo-bar'),
            (Token.SYMBOL, 'foo/bar'),
            (Token.SYMBOL, './foo'),
            (Token.SYMBOL, 'foo_bar'),
            (Token.SYMBOL, 'Foo'), NL,
            EOF,
        ],
    )


def test_invalid_character():
    check_errors(
        'foo $ bar',
        ["Wrong character '$'"],
    )


def test_string():
    check_tokens(
        '"foo" "foo.bar" "foo-bar" "foo/bar" "foo_bar" "Foo" '
        '":foo" "#foo" "{}" "[]" "()" "123"',
        [
            (Token.STRING, 'foo'),
            (Token.STRING, 'foo.bar'),
            (Token.STRING, 'foo-bar'),
            (Token.STRING, 'foo/bar'),
            (Token.STRING, 'foo_bar'),
            (Token.STRING, 'Foo'),
            (Token.STRING, ':foo'),
            (Token.STRING, '#foo'),
            (Token.STRING, '{}'),
            (Token.STRING, '[]'),
            (Token.STRING, '()'),
            (Token.STRING, '123'), NL,
            EOF,
        ],
    )


def test_incomplete_string():
    check_errors(
        '"foo',
        ['String does not and at EOF'],
    )


def test_string_with_newline():
    check_errors(
        '"foo\nbar"',
        ['Newlines are not allowed in strings'],
    )


def test_number():
    check_tokens(
        '1 2.3 4.5d 6d.7 8d 9...',
        [
            (Token.NUMBER, '1'),
            (Token.NUMBER, '2.3'),
            (Token.NUMBER, '4.5d'),
            (Token.NUMBER, '6d.7'),
            (Token.NUMBER, '8d'),
            (Token.NUMBER, '9...'), NL,
            EOF,
        ],
    )


def test_keyword():
    check_tokens(
        ':foo :foo-bar :foo_bar',
        [
            (Token.KEYWORD, 'foo'),
            (Token.KEYWORD, 'foo-bar'),
            (Token.KEYWORD, 'foo_bar'), NL,
            EOF,
        ],
    )


def test_placeholder():
    check_tokens(
        '#foo #foo-bar #foo_bar #foo.bar',
        [
            (Token.PLACEHOLDER, 'foo'),
            (Token.PLACEHOLDER, 'foo-bar'),
            (Token.PLACEHOLDER, 'foo_bar'),
            (Token.PLACEHOLDER, 'foo.bar'), NL,
            EOF,
        ],
    )


def test_brackets():
    check_tokens(
{'fraction_non_alphanumeric': 0.16503077690900017, 'fraction_numerical': 0.008817168524371985, 'mean_word_length': 1.7540082455336692, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3708311', 'n_tokens_mistral': 2026, 'n_tokens_neox': 1956, 'n_words': 428}