<commit_before>/*global define*/

/**
 * @author <NAME> <<EMAIL>>
 * @copyright Apestaartje <http://apestaartje.info>
 */

define(
    [
        'lib/lang/class',
        'lib/dsl/lexer',
        'lib/dsl/token'
    ],
    function (
        Class,
        Lexer,
        Token
    ) {
        'use strict';

        var ListLexer;

        ListLexer = Class.createClass({
            /**
             * @param {string} input
             */
            initialize: function (input) {
                Class.callSuper(Lexer, 'initialize', [input], this);
            },

            /**
             * @param {number} tokenType
             * @returns {string}
             */
            getTokenName: function (tokenType) {
                return ListLexer.tokenNames[tokenType];
            },

            /**
             * @returns {boolean}
             */
            isLetter: function () {
                return (this.c >= 'a' && this.c <= 'x' || this.c >= 'A' && this.c <= 'X');
            },

            /**
             * @returns {boolean}
             */
            isWhitespace: function () {
                return /\s/.test(this.c);
            },

            /**
             * @returns Token
             */
            nextToken: function () {
                var token;

                while (this.c !== Lexer.EOF) {
                    if (this.isWhitespace()) {
                        this.whitespace();
                        continue;
                    } else if (',' === this.c) {
                        this.consume();
                        token = new Token(ListLexer.COMMA, ',');
                    } else if ('[' === this.c) {
                        this.consume();
                        token = new Token(ListLexer.LBRACK, '[');
                    } else if (']' === this.c) {
                        this.consume();
                        token = new Token(ListLexer.RBRACK, ']');
                    } else {
                        if (this.isLetter()) {
                            token = this.name();
                        } else {
                            throw 'Invalid character "' + this.c + '"';
                        }
                    }

                    if (undefined !== token) {
                        break;
                    }
                }

                if (undefined === token) {
                    token = new Token(Lexer.EOF, '<EOF>');
                }

                return token;
            },

            whitespace: function () {
                while (this.isWhitespace()) {
                    this.consume();
                }
            },

            name: function () {
                var buffer = '';

                do {
                    buffer += this.c;
                    this.consume();
                } while (this.isLetter());

                return new Token(ListLexer.NAME, buffer);
            }
        }, Lexer);<commit_msg>pass the lexer to the token constructor<commit_after>/*global define*/

/**
 * @author <NAME> <<EMAIL>>
 * @copyright Apestaartje <http://apestaartje.info>
 */

define(
    [
        'lib/lang/class',
        'lib/dsl/lexer',
        'lib/dsl/token'
    ],
    function (
        Class,
        Lexer,
        Token
    ) {
        'use strict';

        var ListLexer;

        ListLexer = Class.createClass({
            /**
             * @param {string} input
             */
            initialize: function (input) {
                Class.callSuper(Lexer, 'initialize', [input], this);
            },

            /**
             * @param {number} tokenType
             * @returns {string}
             */
            getTokenName: function (tokenType) {
                return ListLexer.tokenNames[tokenType];
            },

            /**
             * @returns {boolean}
             */
            isLetter: function () {
                return (this.c >= 'a' && this.c <= 'x' || this.c >= 'A' && this.c <= 'X');
            },

            /**
             * @returns {boolean}
             */
            isWhitespace: function () {
                return /\s/.test(this.c);
            },

            /**
             * @returns Token
             */
            nextToken: function () {
                var token;

                while (this.c !== Lexer.EOF) {
                    if (this.isWhitespace()) {
                        this.whitespace();
                        continue;
                    } else if (',' === this.c) {
                        this.consume();
                        token = new Token(ListLexer.COMMA, ',', this);
                    } else if ('[' === this.c) {
                        this.consume();
                        token = new Token(ListLexer.LBRACK, '[', this);
                    } else if (']' === this.c) {
                        this.consume();
                        token = new Token(ListLexer.RBRACK, ']', this);
                    } else {
                        if (this.isLetter()) {
                            token = this.name();
                        } else {
                            throw 'Invalid character "' + this.c + '"';
                        }
                    }

                    if (undefined !== token) {
                        break;
                    }
                }

                if (undefined === token) {
                    token = new Token(Lexer.EOF, '<EOF>', this);
                }

                return token;
            },

            whitespace: function () {
                while (this.isWhitespace()) {
                    this.consume();
                }
            },

            name: function () {
                var buffer = '';

                do {
                    buffer += this.c;
                    this.consume();
                } while (this.isLetter());

                return new Token(ListLexer.NAME, buffer, this);
            }
        }, Lexer);
{'fraction_non_alphanumeric': 0.1297722705961152, 'fraction_numerical': 0.0, 'mean_word_length': 0.8344594594594594, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '680092', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1429, 'n_words': 286}