<commit_before>(function(exports) {
    var IDENTIFIER = /^[a-zA-Z$_][a-zA-Z0-9$_]*/;
    var NUMBER = /^-?[0-9]+(\.[0-9]+)?/;
    var COMMENT = /^\/\/.*/;
    var WHITESPACE = /^[^\n\S]+/;
    var INDENT = /^(?:\n[^\n\S]*)+/;

    var chunk;
    var indent;
    var indents;
    var tokens;

    var identifierToken = function() {
        var value,
            name,
            token = IDENTIFIER.exec(chunk);
        if(token) {
            value = token[0];
            switch(value) {
            case 'true':
            case 'false':
                name = 'BOOLEAN';
                break;
            case 'let':
            case 'fn':
            case 'if':
            case 'then':
            case 'else':
            case 'data':
            case 'match':
            case 'case':
            case 'bind':
            case 'return':
                name = value.toUpperCase();
                break;
            default:
                name = 'IDENTIFIER'
                break;
            }
            tokens.push([name, value]);
            return token[0].length;
        }

        return 0;
    };

    var numberToken = function() {
        var token = NUMBER.exec(chunk);
        if(token) {
            tokens.push(['NUMBER', token[0]]);
            return token[0].length;
        }

        return 0;
    };

    var stringToken = function() {
        var firstChar = chunk.charAt(0),
            quoted = false,
            nextChar;
        if(firstChar == '"' || firstChar == "'") {
            for(var i = 1; i < chunk.length; i++) {
                if(!quoted) {
                    nextChar = chunk.charAt(i);
                    if(nextChar == "\\") {
                        quoted = true;
                    } else if(nextChar == firstChar) {
                        tokens.push(['STRING', chunk.substring(0, i + 1)]);
                        return i + 1;
                    }
                } else {
                    quoted = false;
                }
            }
        }

        return 0;
    };

    var commentToken = function() {
        var token = COMMENT.exec(chunk);
        if(token) {
            tokens.push(['COMMENT', token[0]]);
            return token[0].length;
        }

        return 0;
    };

    var whitespaceToken = function() {
        var token = WHITESPACE.exec(chunk);
        if(token) {
            return token[0].length;
        }

        return 0;
    };

    var lineToken = function() {
        var token = INDENT.exec(chunk);
        if(token) {
            var lastNewline = token[0].lastIndexOf("\n") + 1;
            var size = token[0].length - lastNewline;
            if(size > indent) {
                indents.push(size);
                tokens.push(['INDENT', size - indent]);
            } else {
                if(size < indent) {
                    var last = indents[indents.length - 1];
                    while(size < last) {
                        tokens.push(['OUTDENT', last - size]);
                        indents.pop();
                        last = indents[indents.length - 1];
                    }
                }
                if(tokens.length >= 2 && tokens[tokens.length - 2][0] == 'TERMINATOR' && tokens[tokens.length - 1][0] == ')') {
                    /* Swap TERMINATOR and ')' */
                    var t = [tokens.pop(), tokens.pop()];
                    tokens.push(t[0]);
                    tokens.push(t[1]);
                } else {
                    tokens.push(['TERMINATOR', token[0].substring(0, lastNewline)]);
                }
            }
            indent = size;
            return token[0].length;
        }

        return 0;
    };

    var literalToken = function() {
        var tag = chunk.slice(0, 1);
        var next;
        switch(tag) {
        case '=':
            var next = chunk.slice(0, 2);
            switch(next) {
            case '==':
                tokens.push([next, next]);
                return 2;
            }
            tokens.push([tag, tag]);
            return 1;
        case ':':
        case '.':
        case ',':
        case '+':
        case '-':
        case '*':
        case '/':
        case '%':
        case '|':
        case '[':
        case ']':
        case '{':
        case '}':
        case '(':
        case ')':
            tokens.push([tag, tag]);
            return 1;
        }

        return 0;
    };

    exports.tokenise = function(source) {
        indent = 0;
        indents = [];
        tokens = [];
        var i = 0;
        while(chunk = source.slice(i)) {
            var diff = identifierToken() || numberToken() || stringToken() || commentToken() || whitespaceToken() || lineToken() || literalToken();
            if(!diff) {
                throw "Couldn't tokenise: " + chunk.substring(0, chunk.indexOf("\n"));;
            }
            i += diff;
        }

        tokens.push(['EOF', '']);

        return tokens;
    };

    if(!module.parent) {
        exports.tokenise([
            "// Testing",
            " ",
            "let x =",
            " 8",
            " if true then 10 else 100",
            " if true then",
            "  false",
            " else",
            "  true",
            " // Inner comment",<commit_msg>Add line number tracking to lexer<commit_after>(function(exports) {
    var IDENTIFIER = /^[a-zA-Z$_][a-zA-Z0-9$_]*/;
    var NUMBER = /^-?[0-9]+(\.[0-9]+)?/;
    var COMMENT = /^\/\/.*/;
    var WHITESPACE = /^[^\n\S]+/;
    var INDENT = /^(?:\n[^\n\S]*)+/;

    var chunk;
    var indent;
    var indents;
    var tokens;
    var lineno;

    var identifierToken = function() {
        var value,
            name,
            token = IDENTIFIER.exec(chunk);
        if(token) {
            value = token[0];
            switch(value) {
            case 'true':
            case 'false':
                name = 'BOOLEAN';
                break;
            case 'let':
            case 'fn':
            case 'if':
            case 'then':
            case 'else':
            case 'data':
            case 'match':
            case 'case':
            case 'return':
                name = value.toUpperCase();
                break;
            default:
                name = 'IDENTIFIER'
                break;
            }
            tokens.push([name, value, lineno]);
            return token[0].length;
        }

        return 0;
    };

    var numberToken = function() {
        var token = NUMBER.exec(chunk);
        if(token) {
            tokens.push(['NUMBER', token[0], lineno]);
            return token[0].length;
        }

        return 0;
    };

    var stringToken = function() {
        var firstChar = chunk.charAt(0),
            quoted = false,
            nextChar;
        if(firstChar == '"' || firstChar == "'") {
            for(var i = 1; i < chunk.length; i++) {
                if(!quoted) {
                    nextChar = chunk.charAt(i);
                    if(nextChar == "\\") {
                        quoted = true;
                    } else if(nextChar == firstChar) {
                        tokens.push(['STRING', chunk.substring(0, i + 1), lineno]);
                        return i + 1;
                    }
                } else {
                    quoted = false;
                }
            }
        }

        return 0;
    };

    var commentToken = function() {
        var token = COMMENT.exec(chunk);
        if(token) {
            tokens.push(['COMMENT', token[0], lineno]);
            return token[0].length;
        }

        return 0;
    };

    var whitespaceToken = function() {
        var token = WHITESPACE.exec(chunk);
        if(token) {
            return token[0].length;
        }

        return 0;
    };

    var lineToken = function() {
        var token = INDENT.exec(chunk);
        if(token) {
            var lastNewline = token[0].lastIndexOf("\n") + 1;
            var size = token[0].length - lastNewline;
            if(size > indent) {
                indents.push(size);
                tokens.push(['INDENT', size - indent, lineno]);
            } else {
                if(size < indent) {
                    var last = indents[indents.length - 1];
                    while(size < last) {
                        tokens.push(['OUTDENT', last - size, lineno]);
                        indents.pop();
                        last = indents[indents.length - 1];
                    }
                }
                if(tokens.length >= 2 && tokens[tokens.length - 2][0] == 'TERMINATOR' && tokens[tokens.length - 1][0] == ')') {
                    /* Swap TERMINATOR and ')' */
                    var t = [tokens.pop(), tokens.pop()];
                    tokens.push(t[0]);
                    tokens.push(t[1]);
                } else {
                    tokens.push(['TERMINATOR', token[0].substring(0, lastNewline), lineno]);
                }
            }
            indent = size;
            return token[0].length;
        }

        return 0;
    };

    var literalToken = function() {
        var tag = chunk.slice(0, 1);
        var next;
        switch(tag) {
        case '=':
            var next = chunk.slice(0, 2);
            switch(next) {
            case '==':
                tokens.push([next, next, lineno]);
                return 2;
            }
            tokens.push([tag, tag, lineno]);
            return 1;
        case ':':
        case '.':
        case ',':
        case '+':
        case '-':
        case '*':
        case '/':
        case '%':
        case '|':
        case '[':
        case ']':
        case '{':
        case '}':
        case '(':
        case ')':
            tokens.push([tag, tag, lineno]);
            return 1;
        }

        return 0;
    };

    exports.tokenise = function(source) {
        indent = 0;
        indents = [];
        tokens = [];
        lineno = 1;
        var i = 0;
        while(chunk = source.slice(i)) {
            var diff = identifierToken() || numberToken() || stringToken() || commentToken() || whitespaceToken() || lineToken() || literalToken();
            if(!diff) {
                throw "Couldn't tokenise: " + chunk.substring(0, chunk.indexOf("\n"));;
            }
            lineno += source.slice(i, i + diff).split('\n').length - 1;
            i += diff;
        }

        tokens.push(['EOF', '', lineno]);

        return tokens;
    };

    if(!module.parent) {
        exports.tokenise([
            "// Testing",
            " ",
            "let x =",
            " 8",
            " if true then 10 else 100",
            " if true then",
            "  false",
            " else",
            "  true",
            " // Inner comment",
{'fraction_non_alphanumeric': 0.14756540359370635, 'fraction_numerical': 0.0106135369146262, 'mean_word_length': 1.0919182083739045, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3544343', 'n_tokens_mistral': 3160, 'n_tokens_neox': 2945, 'n_words': 642}