<commit_before>// This data model is fairly similar to that used by METAMATH.C, although we
// make statement-level comments their own kind of statement.

// hot inner loop, avoid stuff that could deopt
function eolScan(string, array, sfirst, send, afirst) {
    //console.log('SCAN',string.length,send-sfirst,sfirst,afirst);
    var slen = string.length;
    while (sfirst < send) {
        var ch = string.charCodeAt(sfirst++);
        if (ch <= 13) {
            if (ch === 10) {
                array[afirst++] = sfirst;
            }
            else if (ch === 13) {
                if (sfirst !== send && string.charCodeAt(sfirst) === 10) {
                    sfirst++; // skip LF after CR; note, sfirst>send is possible now
                }
                array[afirst++] = sfirst;
            }
        }
    }

    return [sfirst, afirst];
}

// get indices of all linebreaks in the string
function getEolArray(string) {
    var out = [], sfirst = 0;

    out.push(new Int32Array([0]));

    var quantum = Math.max(100, Math.floor(string.length / 100));
    while (sfirst < string.length) {
        var alen = 2*quantum;
        var acur = new Int32Array(alen);
        var aix  = 0;

        while (sfirst < string.length && (alen - aix) >= quantum) {
            var tmp = eolScan(string, acur, sfirst, sfirst + Math.min(string.length - sfirst, alen - aix), aix);
            sfirst = tmp[0];
            aix = tmp[1];
        }

        if (aix) out.push(acur.subarray(0,aix));
    }

    return out;
}

var SP = []; while (SP.length < 33) SP.push(false); SP[32] = SP[9] = SP[13] = SP[12] = SP[10] = true;

export class MMOMSource {
    constructor(name, string) {
        this.name = name;
        this.text = string;
        this.failed = false;
        this.eolMaps = null;
    }

    lookupPos(pos) {
        if (!this.eolMaps) this.eolMaps = getEolArray(this.text);
        var maps = this.eolMaps;

        var skip = 0;
        var low = 0, high = maps.length - 1;

        while (low !== high) {
            var mid = (low + high + 1) >> 1; // >low, <=high
            if (pos >= maps[mid][0]) {
                while (low < mid) {
                    skip += maps[low++].length;
                }
            }
            else {
                high = mid-1;
            }
        }

        var the_map = maps[low];
        low = 0; high = the_map.length-1;

        while (low !== high) {
            var mid = (low + high + 1) >> 1;
            if (pos >= the_map[mid]) {
                low = mid;
            }
            else {
                high = mid-1;
            }
        }

        return [ low+skip+1, pos-the_map[low]+1 ]; // "column" will be slightly off in the presence of tabs (evil) and supplementary characters (I care, but not much)
    }

    getLine(lnum) {
        if (!this.eolMaps) this.eolMaps = getEolArray(this.text);
        var maps = this.eolMaps;
        lnum--;
        if (lnum < 0) return "";
        var i = 0;
        // could use binary search, but this array has O(1) bounded size
        while (i < maps.length && lnum >= maps[i].length) {
            lnum -= maps[i].length;
            i++;
        }

        if (i === maps.length) return "";
        return this.text.substring(maps[i][lnum], (lnum + 1 === maps[i].length) ? (i+1 === maps.length ? this.text.length : maps[i+1][0]) : maps[i][lnum+1]);
    }

    // keep this in sync with MMOMScanner.getToken
    tokenEnd(pos) {
        var ch;
        while (pos < this.text.length && ((ch = this.text.charCodeAt(pos)) > 32 || !SP[ch])) pos++;
        return pos;
    }
}

// TODO: Add a length field and turn the zones into a linked list to support raw-text extraction without a reparse.  Will probably be needed for efficient comment rendering and WRITE SOURCE
export class MMOMStatement {
    constructor() {
        this.type = MMOMStatement.EOF;
        this._pos = null;
        this.label = null;
        this.math = null;
        this.proof = null;
        this.reparse_zone = null;
        this.reparse_index = 0;
        this.length = 0;
        this.index = 0;
    }

    get raw() {
        var out = '', spans = this.spans;
        for (var i = 0; i < spans.length; i += 3) {
            out += spans[i].text.substr(spans[i+1],spans[i+2]);
        }
        return out;
    }

    get mathPos() { if (!this._pos) this._unlazy(); return this._pos.mathPos; }
    get proofPos() { if (!this._pos) this._unlazy(); return this._pos.proofPos; }
    get startPos() { if (!this._pos) this._unlazy(); return this._pos.startPos; }
    get spans() { if (!this._pos) this._unlazy(); return this._pos.spans; }
    get database() {
        return this.index >= 0 ? this.reparse_zone.database : null;<commit_msg>fix raw-text breakage for statements not at the beginning of file<commit_after>// This data model is fairly similar to that used by METAMATH.C, although we
// make statement-level comments their own kind of statement.

// hot inner loop, avoid stuff that could deopt
function eolScan(string, array, sfirst, send, afirst) {
    //console.log('SCAN',string.length,send-sfirst,sfirst,afirst);
    var slen = string.length;
    while (sfirst < send) {
        var ch = string.charCodeAt(sfirst++);
        if (ch <= 13) {
            if (ch === 10) {
                array[afirst++] = sfirst;
            }
            else if (ch === 13) {
                if (sfirst !== send && string.charCodeAt(sfirst) === 10) {
                    sfirst++; // skip LF after CR; note, sfirst>send is possible now
                }
                array[afirst++] = sfirst;
            }
        }
    }

    return [sfirst, afirst];
}

// get indices of all linebreaks in the string
function getEolArray(string) {
    var out = [], sfirst = 0;

    out.push(new Int32Array([0]));

    var quantum = Math.max(100, Math.floor(string.length / 100));
    while (sfirst < string.length) {
        var alen = 2*quantum;
        var acur = new Int32Array(alen);
        var aix  = 0;

        while (sfirst < string.length && (alen - aix) >= quantum) {
            var tmp = eolScan(string, acur, sfirst, sfirst + Math.min(string.length - sfirst, alen - aix), aix);
            sfirst = tmp[0];
            aix = tmp[1];
        }

        if (aix) out.push(acur.subarray(0,aix));
    }

    return out;
}

var SP = []; while (SP.length < 33) SP.push(false); SP[32] = SP[9] = SP[13] = SP[12] = SP[10] = true;

export class MMOMSource {
    constructor(name, string) {
        this.name = name;
        this.text = string;
        this.failed = false;
        this.eolMaps = null;
    }

    lookupPos(pos) {
        if (!this.eolMaps) this.eolMaps = getEolArray(this.text);
        var maps = this.eolMaps;

        var skip = 0;
        var low = 0, high = maps.length - 1;

        while (low !== high) {
            var mid = (low + high + 1) >> 1; // >low, <=high
            if (pos >= maps[mid][0]) {
                while (low < mid) {
                    skip += maps[low++].length;
                }
            }
            else {
                high = mid-1;
            }
        }

        var the_map = maps[low];
        low = 0; high = the_map.length-1;

        while (low !== high) {
            var mid = (low + high + 1) >> 1;
            if (pos >= the_map[mid]) {
                low = mid;
            }
            else {
                high = mid-1;
            }
        }

        return [ low+skip+1, pos-the_map[low]+1 ]; // "column" will be slightly off in the presence of tabs (evil) and supplementary characters (I care, but not much)
    }

    getLine(lnum) {
        if (!this.eolMaps) this.eolMaps = getEolArray(this.text);
        var maps = this.eolMaps;
        lnum--;
        if (lnum < 0) return "";
        var i = 0;
        // could use binary search, but this array has O(1) bounded size
        while (i < maps.length && lnum >= maps[i].length) {
            lnum -= maps[i].length;
            i++;
        }

        if (i === maps.length) return "";
        return this.text.substring(maps[i][lnum], (lnum + 1 === maps[i].length) ? (i+1 === maps.length ? this.text.length : maps[i+1][0]) : maps[i][lnum+1]);
    }

    // keep this in sync with MMOMScanner.getToken
    tokenEnd(pos) {
        var ch;
        while (pos < this.text.length && ((ch = this.text.charCodeAt(pos)) > 32 || !SP[ch])) pos++;
        return pos;
    }
}

// TODO: Add a length field and turn the zones into a linked list to support raw-text extraction without a reparse.  Will probably be needed for efficient comment rendering and WRITE SOURCE
export class MMOMStatement {
    constructor() {
        this.type = MMOMStatement.EOF;
        this._pos = null;
        this.label = null;
        this.math = null;
        this.proof = null;
        this.reparse_zone = null;
        this.reparse_index = 0;
        this.length = 0;
        this.index = 0;
    }

    get raw() {
        var out = '', spans = this.spans;
        for (var i = 0; i < spans.length; i += 3) {
            out += spans[i].text.substring(spans[i+1],spans[i+2]);
        }
        return out;
    }

    get mathPos() { if (!this._pos) this._unlazy(); return this._pos.mathPos; }
    get proofPos() { if (!this._pos) this._unlazy(); return this._pos.proofPos; }
    get startPos() { if (!this._pos) this._unlazy(); return this._pos.startPos; }
    get spans() { if (!this._pos) this._unlazy(); return this._pos.spans; }
    get database() {
        return this.index >= 0 ? this.reparse_zone.database : null;
{'fraction_non_alphanumeric': 0.14123700514543736, 'fraction_numerical': 0.01428121390318177, 'mean_word_length': 1.8011764705882354, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5120469', 'n_tokens_mistral': 3194, 'n_tokens_neox': 3022, 'n_words': 853}