<commit_before>import preprocess from '../preprocess';

import Input from 'postcss/lib/input';
import test  from 'ava';

function run(t, lines, result) {
    t.same(preprocess(new Input(''), lines), result);
}

test('separates indent from other tokens', t => {
    run(t, [[['space', '  '], ['word', 'ab']]], [
        {
            number:    1,
            indent:    '  ',
            tokens:    [['word', 'ab']],
            colon:     false,
            atrule:    false,
            comment:   false,
            lastComma: false
        }
    ]);
});

test('works with indentless strings', t => {
    run(t, [[['word', 'ab']]], [
        {
            number:    1,
            indent:    '',
            tokens:    [['word', 'ab']],
            colon:     false,
            atrule:    false,
            comment:   false,
            lastComma: false
        }
    ]);
});

test('collects line number', t => {
    run(t, [
        [['newline', '\n', 1]],
        [['newline', '\n', 2]]
    ], [
        {
            number:    1,
            indent:    '',
            tokens:    [['newline', '\n', 1]],
            colon:     false,
            atrule:    false,
            comment:   false,
            lastComma: false
        }, {
            number:    2,
            indent:    '',
            tokens:    [['newline', '\n', 2]],
            colon:     false,
            atrule:    false,
            comment:   false,
            lastComma: false
        }
    ]);
});

test('detects at-rules', t => {
    run(t, [[['at-word', '@ab'], ['space', ' ']]], [
        {
            number:    1,
            indent:    '',
            tokens:    [['at-word', '@ab'], ['space', ' ']],
            colon:     false,
            atrule:    true,
            comment:   false,
            lastComma: false
        }
    ]);
});

test('detects last comma', t => {
    run(t, [[['word', 'ab'], [',', ',']]], [
        {
            number:    1,
            indent:    '',
            tokens:    [['word', 'ab'], [',', ',']],
            colon:     false,
            atrule:    false,
            comment:   false,
            lastComma: true
        }
    ]);
});

test('detects last comma with trailing spaces', t => {
    run(t, [[['word', 'ab'], [',', ','], ['space', ' ']]], [
        {
            number:    1,
            indent:    '',
            tokens:    [['word', 'ab'], [',', ','], ['space', ' ']],
            colon:     false,
            atrule:    false,
            comment:   false,
            lastComma: true
        }
    ]);
});

test('ignore comma inside', t => {
    run(t, [[['word', 'ab'], [',', ','], ['word', 'ba']]], [
        {
            number:    1,
            indent:    '',
            tokens:    [['word', 'ab'], [',', ','], ['word', 'ba']],
            colon:     false,
            atrule:    false,
            comment:   false,
            lastComma: false
        }
    ]);
});

test('detects colon', t => {
    run(t, [[['word', 'ab'], [':', ':'], ['word', 'ba']]], [
        {
            number:    1,
            indent:    '',
            tokens:    [['word', 'ab'], [':', ':'], ['word', 'ba']],
            colon:     true,
            atrule:    false,
            comment:   false,
            lastComma: false
        }
    ]);
});

test('detects comments', t => {
    run(t, [[['comment', '// a']]], [
        {
            number:    1,
            indent:    '',
            tokens:    [['comment', '// a']],
            colon:     false,
            atrule:    false,
            comment:   true,
            lastComma: false<commit_msg>Clean up preprocessor test<commit_after>import preprocess from '../preprocess';

import Input from 'postcss/lib/input';
import test  from 'ava';

function run(t, lines, result) {
    t.same(preprocess(new Input(''), lines), result);
}

let defaults = {
    number:    1,
    indent:    '',
    colon:     false,
    atrule:    false,
    comment:   false,
    lastComma: false
}

test('separates indent from other tokens', t => {
    run(t, [[['space', '  '], ['word', 'ab']]], [
        {
            ...defaults,
            indent: '  ',
            tokens: [['word', 'ab']]
        }
    ]);
});

test('works with indentless strings', t => {
    run(t, [[['word', 'ab']]], [
        {
            ...defaults,
            indent: '',
            tokens: [['word', 'ab']],
        }
    ]);
});

test('collects line number', t => {
    run(t, [
        [['newline', '\n', 1]],
        [['newline', '\n', 2]]
    ], [
        {
            ...defaults,
            number: 1,
            tokens: [['newline', '\n', 1]]
        }, {
            ...defaults,
            number: 2,
            tokens: [['newline', '\n', 2]]
        }
    ]);
});

test('detects at-rules', t => {
    run(t, [[['at-word', '@ab'], ['space', ' ']]], [
        {
            ...defaults,
            tokens: [['at-word', '@ab'], ['space', ' ']],
            atrule: true
        }
    ]);
});

test('detects last comma', t => {
    run(t, [[['word', 'ab'], [',', ',']]], [
        {
            ...defaults,
            tokens:    [['word', 'ab'], [',', ',']],
            lastComma: true
        }
    ]);
});

test('detects last comma with trailing spaces', t => {
    run(t, [[['word', 'ab'], [',', ','], ['space', ' ']]], [
        {
            ...defaults,
            tokens:    [['word', 'ab'], [',', ','], ['space', ' ']],
            lastComma: true
        }
    ]);
});

test('ignore comma inside', t => {
    run(t, [[['word', 'ab'], [',', ','], ['word', 'ba']]], [
        {
            ...defaults,
            tokens:    [['word', 'ab'], [',', ','], ['word', 'ba']],
            lastComma: false
        }
    ]);
});

test('detects colon', t => {
    run(t, [[['word', 'ab'], [':', ':'], ['word', 'ba']]], [
        {
            ...defaults,
            tokens:    [['word', 'ab'], [':', ':'], ['word', 'ba']],
            colon:     true
        }
    ]);
});

test('detects comments', t => {
    run(t, [[['comment', '// a']]], [
        {
            ...defaults,
            tokens:  [['comment', '// a']],
            comment: true
{'fraction_non_alphanumeric': 0.2249918540241121, 'fraction_numerical': 0.003421309872922776, 'mean_word_length': 1.3342205323193916, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3469394', 'n_tokens_mistral': 1938, 'n_tokens_neox': 1927, 'n_words': 423}