<commit_before>/*jshint bitwise: true, camelcase: true, curly: true, eqeqeq: true, forin: true, immed: true, indent: 4, latedef: true, newcap: true, noarg: true, noempty: true, nonew: true, regexp: true, undef: true, unused: true, strict: true*/
/*global define*/
define(["util"], function (util) {

    "use strict";

    function error(msg) {
        return function (state) {
            throw new Error(msg + " (line " + state.lineNum + ", char " + state.charNum + ")");
        };
    }
    
    var createForm = util.createForm;

    function readList(state) {
        var origState = state.clone();
        var openChr = state.input.charAt(state.i);
        var closeChr = openChr === "(" ? ")" :
                       openChr === "[" ? "]" :
                       openChr === "{" ? "}" :
                       error("Unknown list opener '" + openChr + "'")(state);
        var sym = openChr === "(" ? "paren" :
                  openChr === "[" ? "bracket" :
                  openChr === "{" ? "brace" : null;
        state.i++;
        var values = [];
        var closed = false;
        consumeWhitespace(state);
        while (state.i < state.l) {
            if (state.input.charAt(state.i) === closeChr) {
                state.i++;
                closed = true;
                break;
            } else {
                values.push(read(state));
                consumeWhitespace(state);
            }
        }
        if (!closed) {
            error("Expected '" + closeChr + "'")(origState);
        }
        return createForm("list", values, { sym: sym });
    }

    function readNum(state) {
        var rxNum = /^[0-9\.]+$/;
        var rxHex = /^0x[0-9A-F]+$/i;
        var chunk = readToDelim(state);
        if (rxNum.test(chunk)) {
            return createForm("literal", parseFloat(chunk, 10));
        } else if (rxHex.test(chunk)) {
            return createForm("literal", parseInt(chunk, 16));
        } else {
            error("Invalid characters in number literal '" + chunk + "'")(state);
        }
    }
    
    var escapeable = util.makeMap(["b", "t", "n", "v", "f", "r", "u", '"', "'", "\\"]);

    function readString(state) {
        state.i++;
        var origState = state.clone();
        var chunk = "";
        var escaped = false;
        var input = state.input;
        while (state.i < state.l) {
            var chr = input.charAt(state.i);
            if (chr === '"' && !escaped) {
                break;
            }
            if (escaped) {
                if (escapeable.hasOwnProperty(chr)) {
                    chunk += "\\" + chr;
                    escaped = false;
                } else {
                    error("Invalid escape sequence '\\" + chr + "'")(state);
                }
            } else if (chr === "\\") {
                escaped = true;
            } else {
                chunk += chr;
            }
            state.i++;
            state.charNum++;
        }
        
        if (state.i === state.l) {
            error("Unclosed string")(origState);
        }
        state.i++;
        return createForm("literal", chunk);
    }
    
    function readBool(state) {
        var chr = state.input.charAt(state.i++);
        return createForm("literal", chr === "t");
    }
    
    function readRegex(state) {
        state.i++;
        var origState = state.clone();
        var chunk = "";
        var escaped = false;
        var input = state.input;
        while (state.i < state.l) {
            var chr = input.charAt(state.i);
            if (chr === '/' && !escaped) {
                break;
            }
            if (escaped) {
                escaped = false;
            } else if (chr === "\\") {
                escaped = true;
            }
            chunk += chr;
            state.i++;
            state.charNum++;
        }
        if (state.i === state.l) {
            error("Unclosed regular expression")(origState);
        }
        state.i++;
        var options = readToDelim(state);
        if (!/^[gim]*$/.test(options)) {
            error("Invalid regular expression options '" + options + "'")(state);
        }
        return createForm("literal", new RegExp(chunk.slice(1, chunk.length), options));
    }
    
    var specials = {
        "t": readBool,
        "f": readBool,
        "/": readRegex
    };
    <commit_msg>RegExp reading tweak<commit_after>/*jshint bitwise: true, camelcase: true, curly: true, eqeqeq: true, forin: true, immed: true, indent: 4, latedef: true, newcap: true, noarg: true, noempty: true, nonew: true, regexp: true, undef: true, unused: true, strict: true*/
/*global define*/
define(["util"], function (util) {

    "use strict";

    function error(msg) {
        return function (state) {
            throw new Error(msg + " (line " + state.lineNum + ", char " + state.charNum + ")");
        };
    }
    
    var createForm = util.createForm;

    function readList(state) {
        var origState = state.clone();
        var openChr = state.input.charAt(state.i);
        var closeChr = openChr === "(" ? ")" :
                       openChr === "[" ? "]" :
                       openChr === "{" ? "}" :
                       error("Unknown list opener '" + openChr + "'")(state);
        var sym = openChr === "(" ? "paren" :
                  openChr === "[" ? "bracket" :
                  openChr === "{" ? "brace" : null;
        state.i++;
        var values = [];
        var closed = false;
        consumeWhitespace(state);
        while (state.i < state.l) {
            if (state.input.charAt(state.i) === closeChr) {
                state.i++;
                closed = true;
                break;
            } else {
                values.push(read(state));
                consumeWhitespace(state);
            }
        }
        if (!closed) {
            error("Expected '" + closeChr + "'")(origState);
        }
        return createForm("list", values, { sym: sym });
    }

    function readNum(state) {
        var rxNum = /^[0-9\.]+$/;
        var rxHex = /^0x[0-9A-F]+$/i;
        var chunk = readToDelim(state);
        if (rxNum.test(chunk)) {
            return createForm("literal", parseFloat(chunk, 10));
        } else if (rxHex.test(chunk)) {
            return createForm("literal", parseInt(chunk, 16));
        } else {
            error("Invalid characters in number literal '" + chunk + "'")(state);
        }
    }
    
    var escapeable = util.makeMap(["b", "t", "n", "v", "f", "r", "u", '"', "'", "\\"]);

    function readString(state) {
        state.i++;
        var origState = state.clone();
        var chunk = "";
        var escaped = false;
        var input = state.input;
        while (state.i < state.l) {
            var chr = input.charAt(state.i);
            if (chr === '"' && !escaped) {
                break;
            }
            if (escaped) {
                if (escapeable.hasOwnProperty(chr)) {
                    chunk += "\\" + chr;
                    escaped = false;
                } else {
                    error("Invalid escape sequence '\\" + chr + "'")(state);
                }
            } else if (chr === "\\") {
                escaped = true;
            } else {
                chunk += chr;
            }
            state.i++;
            state.charNum++;
        }
        
        if (state.i === state.l) {
            error("Unclosed string")(origState);
        }
        state.i++;
        return createForm("literal", chunk);
    }
    
    function readBool(state) {
        var chr = state.input.charAt(state.i++);
        return createForm("literal", chr === "t");
    }
    
    function readRegex(state) {
        state.i++;
        var origState = state.clone();
        var chunk = "";
        var escaped = false;
        var input = state.input;
        while (state.i < state.l) {
            var chr = input.charAt(state.i);
            if (chr === '/' && !escaped) {
                break;
            }
            if (escaped) {
                escaped = false;
            } else if (chr === "\\") {
                escaped = true;
            }
            chunk += chr;
            state.i++;
            state.charNum++;
        }
        if (state.i === state.l) {
            error("Unclosed regular expression")(origState);
        }
        state.i++;
        var options = readToDelim(state);
        if (!/^[gim]*$/.test(options)) {
            error("Invalid regular expression options '" + options + "'")(state);
        }
        return createForm("literal", new RegExp(chunk, options));
    }
    
    var specials = {
        "t": readBool,
        "f": readBool,
        "/": readRegex
    };
    
{'fraction_non_alphanumeric': 0.16019751952227837, 'fraction_numerical': 0.002411575562700965, 'mean_word_length': 1.421857619577308, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4464848', 'n_tokens_mistral': 2591, 'n_tokens_neox': 2507, 'n_words': 575}