<commit_before>import tokenize from '../lib/tokenize';
import Input    from '../lib/input';

import { expect } from 'chai';

let test = (css, opts, tokens) => {
    if ( typeof tokens === 'undefined' ) [tokens, opts] = [opts, tokens];
    expect(tokenize(new Input(css, opts))).to.eql(tokens);
};

describe('tokenize', () => {

    it('tokenizes empty file', () => {
        test('', []);
    });

    it('tokenizes space', () => {
        test('\r\n \f\t', [ ['space', '\r\n \f\t'] ]);
    });

    it('tokenizes word', () => {
        test('ab', [ ['word', 'ab', 1, 1, 1, 2] ]);
    });

    it('splits word by !', () => {
        test('aa!bb', [
            ['word', 'aa',  1, 1, 1, 2],
            ['word', '!bb', 1, 3, 1, 5]
        ]);
    });

    it('changes lines in spaces', () => {
        test('a \n b', [
            ['word',  'a', 1, 1, 1, 1],
            ['space', ' \n '],
            ['word',  'b', 2, 2, 2, 2]
        ]);
    });

    it('tokenizes control chars', () => {
        test('{:;}', [
            ['{', '{', 1, 1],
            [':', ':', 1, 2],
            [';', ';', 1, 3],
            ['}', '}', 1, 4]
        ]);
    });

    it('escapes control symbols', () => {
        test('\\(\\{\\"\\@\\\\""', [
            ['word',   '\\(',  1, 1, 1, 2],
            ['word',   '\\{',  1, 3, 1, 4],
            ['word',   '\\"',  1, 5, 1, 6],
            ['word',   '\\@',  1, 7, 1, 8],
            ['word',   '\\\\', 1, 9, 1, 10],
            ['string', '""',  1, 11, 1, 12]
        ]);
    });

    it('escapes backslash', () => {
        test('\\\\\\\\{', [
            ['word', '\\\\\\\\', 1, 1, 1, 4],
            ['{',    '{',        1, 5]
        ]);<commit_msg>Better test for escaping<commit_after>import tokenize from '../lib/tokenize';
import Input    from '../lib/input';

import { expect } from 'chai';

let test = (css, opts, tokens) => {
    if ( typeof tokens === 'undefined' ) [tokens, opts] = [opts, tokens];
    expect(tokenize(new Input(css, opts))).to.eql(tokens);
};

describe('tokenize', () => {

    it('tokenizes empty file', () => {
        test('', []);
    });

    it('tokenizes space', () => {
        test('\r\n \f\t', [ ['space', '\r\n \f\t'] ]);
    });

    it('tokenizes word', () => {
        test('ab', [ ['word', 'ab', 1, 1, 1, 2] ]);
    });

    it('splits word by !', () => {
        test('aa!bb', [
            ['word', 'aa',  1, 1, 1, 2],
            ['word', '!bb', 1, 3, 1, 5]
        ]);
    });

    it('changes lines in spaces', () => {
        test('a \n b', [
            ['word',  'a', 1, 1, 1, 1],
            ['space', ' \n '],
            ['word',  'b', 2, 2, 2, 2]
        ]);
    });

    it('tokenizes control chars', () => {
        test('{:;}', [
            ['{', '{', 1, 1],
            [':', ':', 1, 2],
            [';', ';', 1, 3],
            ['}', '}', 1, 4]
        ]);
    });

    it('escapes control symbols', () => {
        test('\\(\\{\\"\\@\\\\""', [
            ['word',   '\\(',  1,  1, 1,  2],
            ['word',   '\\{',  1,  3, 1,  4],
            ['word',   '\\"',  1,  5, 1,  6],
            ['word',   '\\@',  1,  7, 1,  8],
            ['word',   '\\\\', 1,  9, 1, 10],
            ['string', '""',   1, 11, 1, 12]
        ]);
    });

    it('escapes backslash', () => {
        test('\\\\\\\\{', [
            ['word', '\\\\\\\\', 1, 1, 1, 4],
            ['{',    '{',        1, 5]
        ]);
{'fraction_non_alphanumeric': 0.29856009403467526, 'fraction_numerical': 0.03585071995298266, 'mean_word_length': 1.4141843971631205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5129234', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1236, 'n_words': 287}