<commit_before>/*
* Copyright (c) 2014 <NAME>, <EMAIL>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
function makeLexerState() {
    return {
        lineNumber :    0,
        charsRead  :    0, // all characters read, excluding the current line
        line       :   "",
        linePos    :    0,
        sym        : null,
        text       :   "",
        startCoord : null,
        set        : function (sym, text) {
            this.sym  = sym;
            this.text = text;
        }
    }
}

function cloneLexerState(old) {
    return {
        lineNumber : old.lineNumber,
        charsRead  : old.charsRead, // all characters read, excluding the current line
        line       : old.line,
        linePos    : old.linePos,
        sym        : old.sym,
        text       : old.text,
        startCoord : old.startCoord,
        set        : function (sym, text) {
            this.sym  = sym;
            this.text = text;
        }
    }
}

function Lexer(fileContent) {
    var SEPARATOR = "----",
        PRIMITIVE = "primitive",
        peekDone  = false,
        state     = makeLexerState(),
        stateAfterPeek = null,
        fileLines = fileContent.split('\n'),
        _this = this;

    function makeSourceCoordinate() {
        var startLine   = state.lineNumber,
            startColumn = state.linePos + 1,
            charIndex   = state.charsRead + state.linePos;

        return {
            get startLine()   { return startLine;   },
            get startColumn() { return startColumn; },
            get charIndex()   { return charIndex;   },
            toString: function() {
                return "SrcCoord(line: " + startLine + ", col: " + startColumn + ")";
            }
        };
    }

    this.getStartCoordinate = function () {
        return state.startCoord;
    };

    this.getSym = function () {
        if (peekDone) {
            peekDone = false;
            state = stateAfterPeek;
            stateAfterPeek = null;
            return state.sym;
        }

        state.startCoord = makeSourceCoordinate();

        do {
            if (!hasMoreInput()) {
                state.set(Sym.NONE, "");
                return state.sym;
            }
            skipWhiteSpace();
            skipComment();
        }
        while (endOfLine() || /\s/.test(currentChar())
            || currentChar() == '"');

        if (currentChar() == '\'') {
            lexString();
        } else if (currentChar() == '[') {
            match(Sym.NewBlock);
        } else if (currentChar() == ']') {
            match(Sym.EndBlock);
        } else if (currentChar() == ':') {
            if (bufchar(state.linePos + 1) == '=') {
                state.linePos += 2;
                state.set(Sym.Assign, ":=");
            } else {
                state.linePos++;
                state.set(Sym.Colon, ":");
            }
        } else if (currentChar() == '(') {
            match(Sym.NewTerm);
        } else if (currentChar() == ')') {
            match(Sym.EndTerm);
        } else if (currentChar() == '#') {
            match(Sym.Pound);
        } else if (currentChar() == '^') {
            match(Sym.Exit);
        } else if (currentChar() == '.') {
            match(Sym.Period);
        } else if (currentChar() == '-') {
            if (state.line.indexOf(SEPARATOR, state.linePos) == state.linePos) {
                state.text = "";
                while (currentChar() == '-') {
                    state.text += bufchar(state.linePos++);
                }
                state.sym = Sym.Separator;
            } else {
                lexOperator();
            }
        } else if (isOperator(currentChar())) {
            lexOperator();
        } else if (nextWordInBufferIs(PRIMITIVE)) {
            state.linePos += PRIMITIVE.length;
            state.set(Sym.Primitive, PRIMITIVE);
        } else if (/[A-Za-z]/.test(currentChar())) {
            state.set(Sym.Identifier, "");
            while (isIdentifierChar(currentChar())) {
                state.text += bufchar(state.linePos++);
            }
            if (bufchar(state.linePos) == ':') {
                state.sym = Sym.Keyword;
                state.linePos++;
                state.text += ':';
                if (/[A-Za-z]/.test(currentChar())) {
                    state.sym = Sym.KeywordSequence;
                    while (/[A-Za-z]/.test(currentChar()) || currentChar() == ':') {
                        state.text += bufchar(state.linePos++);
                    }
                }
            }
        } else if (/\d/.test(currentChar())) {
            lexNumber();
        } else {
            state.set(Sym.NONE, currentChar());
        }

        return state.sym;
    };

    function lexNumber() {
        state.set(Sym.Integer, "");

        var sawDecimalMark = false;

        do {
            state.text += bufchar(state.linePos++);

            if (!sawDecimalMark      &&
                '.' == currentChar() &&
                /\d/.test(bufchar(state.linePos + 1))) {
                state.sym = Sym.Double;
                state.text += bufchar(state.linePos++);
            }
        } while (/\d/.test(currentChar()));
    }

    function lexString() {
        state.set(Sym.STString, "");

        do {
            state.text += bufchar(++state.linePos);
        }
        while (currentChar() != '\'');

        state.text = state.text.slice(0, -1);
        state.linePos++;
    }

    function lexOperator() {
        if (isOperator(bufchar(state.linePos + 1))) {
            state.set(Sym.OperatorSequence, "");
            while (isOperator(currentChar())) {
                state.text += bufchar(state.linePos++);
            }
        } else if (currentChar() == '~') {
            match(Sym.Not);<commit_msg>Add support for string escape characters<commit_after>/*
* Copyright (c) 2014 <NAME>, <EMAIL>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
function makeLexerState() {
    return {
        lineNumber :    0,
        charsRead  :    0, // all characters read, excluding the current line
        line       :   "",
        linePos    :    0,
        sym        : null,
        text       :   "",
        startCoord : null,
        set        : function (sym, text) {
            this.sym  = sym;
            this.text = text;
        }
    }
}

function cloneLexerState(old) {
    return {
        lineNumber : old.lineNumber,
        charsRead  : old.charsRead, // all characters read, excluding the current line
        line       : old.line,
        linePos    : old.linePos,
        sym        : old.sym,
        text       : old.text,
        startCoord : old.startCoord,
        set        : function (sym, text) {
            this.sym  = sym;
            this.text = text;
        }
    }
}

function Lexer(fileContent) {
    var SEPARATOR = "----",
        PRIMITIVE = "primitive",
        peekDone  = false,
        state     = makeLexerState(),
        stateAfterPeek = null,
        fileLines = fileContent.split('\n'),
        _this = this;

    function makeSourceCoordinate() {
        var startLine   = state.lineNumber,
            startColumn = state.linePos + 1,
            charIndex   = state.charsRead + state.linePos;

        return {
            get startLine()   { return startLine;   },
            get startColumn() { return startColumn; },
            get charIndex()   { return charIndex;   },
            toString: function() {
                return "SrcCoord(line: " + startLine + ", col: " + startColumn + ")";
            }
        };
    }

    this.getStartCoordinate = function () {
        return state.startCoord;
    };

    this.getSym = function () {
        if (peekDone) {
            peekDone = false;
            state = stateAfterPeek;
            stateAfterPeek = null;
            return state.sym;
        }

        state.startCoord = makeSourceCoordinate();

        do {
            if (!hasMoreInput()) {
                state.set(Sym.NONE, "");
                return state.sym;
            }
            skipWhiteSpace();
            skipComment();
        }
        while (endOfLine() || /\s/.test(currentChar())
            || currentChar() == '"');

        if (currentChar() == '\'') {
            lexString();
        } else if (currentChar() == '[') {
            match(Sym.NewBlock);
        } else if (currentChar() == ']') {
            match(Sym.EndBlock);
        } else if (currentChar() == ':') {
            if (bufchar(state.linePos + 1) == '=') {
                state.linePos += 2;
                state.set(Sym.Assign, ":=");
            } else {
                state.linePos++;
                state.set(Sym.Colon, ":");
            }
        } else if (currentChar() == '(') {
            match(Sym.NewTerm);
        } else if (currentChar() == ')') {
            match(Sym.EndTerm);
        } else if (currentChar() == '#') {
            match(Sym.Pound);
        } else if (currentChar() == '^') {
            match(Sym.Exit);
        } else if (currentChar() == '.') {
            match(Sym.Period);
        } else if (currentChar() == '-') {
            if (state.line.indexOf(SEPARATOR, state.linePos) == state.linePos) {
                state.text = "";
                while (currentChar() == '-') {
                    state.text += bufchar(state.linePos++);
                }
                state.sym = Sym.Separator;
            } else {
                lexOperator();
            }
        } else if (isOperator(currentChar())) {
            lexOperator();
        } else if (nextWordInBufferIs(PRIMITIVE)) {
            state.linePos += PRIMITIVE.length;
            state.set(Sym.Primitive, PRIMITIVE);
        } else if (/[A-Za-z]/.test(currentChar())) {
            state.set(Sym.Identifier, "");
            while (isIdentifierChar(currentChar())) {
                state.text += bufchar(state.linePos++);
            }
            if (bufchar(state.linePos) == ':') {
                state.sym = Sym.Keyword;
                state.linePos++;
                state.text += ':';
                if (/[A-Za-z]/.test(currentChar())) {
                    state.sym = Sym.KeywordSequence;
                    while (/[A-Za-z]/.test(currentChar()) || currentChar() == ':') {
                        state.text += bufchar(state.linePos++);
                    }
                }
            }
        } else if (/\d/.test(currentChar())) {
            lexNumber();
        } else {
            state.set(Sym.NONE, currentChar());
        }

        return state.sym;
    };

    function lexNumber() {
        state.set(Sym.Integer, "");

        var sawDecimalMark = false;

        do {
            state.text += bufchar(state.linePos++);

            if (!sawDecimalMark      &&
                '.' == currentChar() &&
                /\d/.test(bufchar(state.linePos + 1))) {
                state.sym = Sym.Double;
                state.text += bufchar(state.linePos++);
            }
        } while (/\d/.test(currentChar()));
    }

    function lexEscapeChar() {
        const current = currentChar();

        switch (current) {
            case 't':  state.text += "\t"; break;
            case 'b':  state.text += "\b"; break;
            case 'n':  state.text += "\n"; break;
            case 'r':  state.text += "\r"; break;
            case 'f':  state.text += "\f"; break;
            case '\'': state.text += "'"; break;
            case '\\': state.text += "\\"; break;
        }
        state.linePos++;
    }

    function lexStringChar() {
        if (currentChar() === '\\') {
            state.linePos++;
            lexEscapeChar();
        } else {
            state.text += currentChar();
            state.linePos++;
        }
    }

    function lexString() {
        state.set(Sym.STString, "");
        state.linePos++

        while (currentChar() != '\'') {
            lexStringChar();
            while (endOfLine()) {
                if (!readNextLine()) { return; }
                state.text += "\n";
            }
        }

        state.linePos++;
    }

    function lexOperator() {
        if (isOperator(bufchar(state.linePos + 1))) {
            state.set(Sym.OperatorSequence, "");
            while (isOperator(currentChar())) {
                state.text += bufchar(state.linePos++);
            }
        } else if (currentChar() == '~') {
            match(Sym.Not);
{'fraction_non_alphanumeric': 0.13465676407302005, 'fraction_numerical': 0.0018046782813909905, 'mean_word_length': 1.5710206995003568, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2819246', 'n_tokens_mistral': 4057, 'n_tokens_neox': 3888, 'n_words': 966}