<commit_before>define(['rpn/tokenizer'], function (tokenizer) {
    'use strict';

    describe('Tokenizer tests', function () {
    });
});<commit_msg>Add Tokenizer tests<commit_after>define(['rpn/tokenizer'], function (tokenizer) {
    'use strict';

    describe('Tokenizer', function () {
        it("should ignore empty strings", function () {
            expect(tokenizer('')).toEqual([]);
            expect(tokenizer('    ')).toEqual([]);
        });

        it("should ignore whitespace", function () {
            expect(tokenizer('   1  2    ')).toEqual(['1', '2']);
        });

        it("should ignore non-operators", function () {
            expect(tokenizer('1noop 2 noop3|4')).toEqual(['1', '2', '3', '4']);
        });

        it("should detect operators", function () {
            expect(tokenizer('+-*x/%pow_')).toEqual(['+', '-', '*', 'x', '/', '%', 'pow', '_']);
        });

        it("should detect mixed-case operators and return lowercase", function () {
            expect(tokenizer('X PoW POW pOw')).toEqual(['x', 'pow', 'pow', 'pow']);
        });

        it("should detect parentheses", function () {
            expect(tokenizer('()')).toEqual(['(', ')']);
        });

        it("should detect floats", function () {
            expect(tokenizer('0.500000000')).toEqual(['0.500000000']);
        });

        it("should NOT detect negative", function () {
            expect(tokenizer('-3')).toEqual(['-', '3']);
        });

        it("should detect mix of numbers operators and  parentheses", function () {
            expect(tokenizer('0.500000000+ -(-3/-2)')).toEqual(['0.500000000', '+', '-', '(', '-', '3', '/', '-', '2', ')']);
        });
    });
});
{'fraction_non_alphanumeric': 0.23307828134196587, 'fraction_numerical': 0.034137728075338436, 'mean_word_length': 2.5051546391752577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '342833', 'n_tokens_mistral': 519, 'n_tokens_neox': 488, 'n_words': 96}