<commit_before>/*jshint bitwise: true, camelcase: true, curly: true, eqeqeq: true, forin: true, immed: true, indent: 4, latedef: true, newcap: true, noarg: true, noempty: true, nonew: true, regexp: true, undef: true, unused: true, strict: true*/
/*global define*/
define(["util"], function (util) {

    "use strict";

    function error(msg) {
        return function (state) {
            throw new Error(msg + " (line " + state.lineNum + ", char " + state.charNum + ")");
        };
    }

    var skip = { toString: function () { return "{skip}"; } };

    function newline(state) {
        state.i++;
        state.lineNum++;
        state.charNum = 1;
        return read(state);
    }

    function readList(state) {
        var origState = state.clone();
        var openChr = state.input.charAt(state.i);
        var closeChr = openChr === "(" ? ")" :
                       openChr === "[" ? "]" :
                       openChr === "{" ? "}" :
                       error("Unknown list opener '" + openChr + "'")(state);
        var sym = openChr === "(" ? "paren" :
                  openChr === "[" ? "bracket" :
                  openChr === "{" ? "brace" : null;
        state.i++;
        var values = [];
        var closed = false;
        while (state.i < state.l) {
            if (state.input.charAt(state.i) === closeChr) {
                state.i++;
                closed = true;
                break;
            } else {
                var s = read(state);
                values.push(s);
            }
        }
        if (!closed) {
            error("Expected '" + closeChr + "'")(origState);
        }
        return util.createForm("list", values, { sym: sym });
    }

    function readNum(state) {
        var rxNum = /^[0-9\.]+$/;
        var rxHex = /^0x[0-9A-F]+$/i;
        var chunk = readToDelim(state);
        if (rxNum.test(chunk)) {
            return util.createForm("literal", parseFloat(chunk, 10));
        } else if (rxHex.test(chunk)) {
            return util.createForm("literal", parseInt(chunk, 16));
        } else {
            error("Invalid characters in number literal '" + chunk + "'")(state);
        }
    }

    function readString(state) {
        state.i++;
        var origState = state.clone();
        
        var chunk = "";
        var lastChr = null;
        var input = state.input;
        while (state.i < state.l) {
            var chr = input.charAt(state.i);
            if (chr === '"' && lastChr !== "\\") {
                break;
            } else {
                chunk += chr;
                lastChr = chr;
                state.i++;
                state.charNum++;
            }
        }
        
        if (state.i === state.l) {
            error("Unclosed string")(origState);
        }
        state.i++;
        return util.createForm("literal", '"' + chunk + '"');
    }
    
    function readSpecial(state) {
        var chunk = readToDelim(state);
        if (chunk === "#t") {
            return util.createForm("literal", true);
        } else if (chunk === "#f") {
            return util.createForm("literal", false);
        }
    }

    function readSymbol(state) {
        // TODO: this might be a tad too relaxed
        return util.createForm("symbol", readToDelim(state));
    }

    function readComment(state) {
        var text = "";
        state.i++;
        while (state.i < state.l) {
            var chr = state.input.charAt(state.i++);
            if (chr === "\n") {
                break;
            } else if (chr !== "\r") {
                text += chr;
            }
        }
        return util.createForm("list", [util.createForm("symbol", "comment"), text]);
    }
<commit_msg>Altered string literals, probably not done yet<commit_after>/*jshint bitwise: true, camelcase: true, curly: true, eqeqeq: true, forin: true, immed: true, indent: 4, latedef: true, newcap: true, noarg: true, noempty: true, nonew: true, regexp: true, undef: true, unused: true, strict: true*/
/*global define*/
define(["util"], function (util) {

    "use strict";

    function error(msg) {
        return function (state) {
            throw new Error(msg + " (line " + state.lineNum + ", char " + state.charNum + ")");
        };
    }

    var skip = { toString: function () { return "{skip}"; } };

    function newline(state) {
        state.i++;
        state.lineNum++;
        state.charNum = 1;
        return read(state);
    }

    function readList(state) {
        var origState = state.clone();
        var openChr = state.input.charAt(state.i);
        var closeChr = openChr === "(" ? ")" :
                       openChr === "[" ? "]" :
                       openChr === "{" ? "}" :
                       error("Unknown list opener '" + openChr + "'")(state);
        var sym = openChr === "(" ? "paren" :
                  openChr === "[" ? "bracket" :
                  openChr === "{" ? "brace" : null;
        state.i++;
        var values = [];
        var closed = false;
        while (state.i < state.l) {
            if (state.input.charAt(state.i) === closeChr) {
                state.i++;
                closed = true;
                break;
            } else {
                var s = read(state);
                values.push(s);
            }
        }
        if (!closed) {
            error("Expected '" + closeChr + "'")(origState);
        }
        return util.createForm("list", values, { sym: sym });
    }

    function readNum(state) {
        var rxNum = /^[0-9\.]+$/;
        var rxHex = /^0x[0-9A-F]+$/i;
        var chunk = readToDelim(state);
        if (rxNum.test(chunk)) {
            return util.createForm("literal", parseFloat(chunk, 10));
        } else if (rxHex.test(chunk)) {
            return util.createForm("literal", parseInt(chunk, 16));
        } else {
            error("Invalid characters in number literal '" + chunk + "'")(state);
        }
    }

    function readString(state) {
        // TODO: escaping probably needs more work
        state.i++;
        var origState = state.clone();
        var chunk = "";
        var escaped = false;
        var input = state.input;
        while (state.i < state.l) {
            var chr = input.charAt(state.i);
            if (chr === '"' && !escaped) {
                break;
            }
            if (escaped || chr !== "\\") {
                chunk += chr;
                escaped = false;
            } else if (chr === "\\") {
                escaped = true;
            }
            state.i++;
            state.charNum++;
        }
        
        if (state.i === state.l) {
            error("Unclosed string")(origState);
        }
        state.i++;
        return util.createForm("literal", chunk);
    }
    
    function readSpecial(state) {
        var chunk = readToDelim(state);
        if (chunk === "#t") {
            return util.createForm("literal", true);
        } else if (chunk === "#f") {
            return util.createForm("literal", false);
        }
    }

    function readSymbol(state) {
        // TODO: this might be a tad too relaxed
        return util.createForm("symbol", readToDelim(state));
    }

    function readComment(state) {
        var text = "";
        state.i++;
        while (state.i < state.l) {
            var chr = state.input.charAt(state.i++);
            if (chr === "\n") {
                break;
            } else if (chr !== "\r") {
                text += chr;
            }
        }
        return util.createForm("list", [util.createForm("symbol", "comment"), text]);
    }

{'fraction_non_alphanumeric': 0.1595562012944129, 'fraction_numerical': 0.0029058248580108308, 'mean_word_length': 1.5206391478029295, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3044321', 'n_tokens_mistral': 2273, 'n_tokens_neox': 2191, 'n_words': 525}