<commit_before>/* -*- Mode: Java; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- /
/* vim: set shiftwidth=4 tabstop=8 autoindent cindent expandtab: */

var ERRORS = 0, WARNINGS = 1, TODOS = 5;
var verbosity = WARNINGS;

function log(msg) {
    if (console && console.log)
        console.log(msg);
    else if (print)
        print(msg);
}

function warn(msg) {
    if (verbosity >= WARNINGS)
        log("Warning: "+ msg);
}

function error(msg) {
    throw new Error(msg);
}

function TODO(what) {
    if (verbosity >= TODOS)
        log("TODO: "+ what);
}

function malformed(msg) {
    error("Malformed PDF: "+ msg);
}

function assert(cond, msg) {
    if (!cond)
        error(msg);
}

// In a well-formed PDF, |cond| holds.  If it doesn't, subsequent
// behavior is undefined.
function assertWellFormed(cond, msg) {
    if (!cond)
        malformed(msg);
}

function shadow(obj, prop, value) {
    Object.defineProperty(obj, prop, { value: value, enumerable: true });
    return value;
}

var Stream = (function() {
    function constructor(arrayBuffer, start, length, dict) {
        this.bytes = new Uint8Array(arrayBuffer);
        this.start = start || 0;
        this.pos = this.start;
        this.end = (start + length) || arrayBuffer.byteLength;
        this.dict = dict;
    }

    constructor.prototype = {
        get length() {
            return this.end - this.start;
        },
        getByte: function() {
            var bytes = this.bytes;
            if (this.pos >= this.end)
                return -1;
            return bytes[this.pos++];
        },
        lookChar: function() {
            var bytes = this.bytes;
            if (this.pos >= this.end)
                return;
            return String.fromCharCode(bytes[this.pos]);
        },
        getChar: function() {
            var ch = this.lookChar();
            if (!ch)
                return ch;
            this.pos++;
            return ch;
        },
        skip: function(n) {
            if (!n)
                n = 1;
            this.pos += n;
        },
        reset: function() {
            this.pos = this.start;
        },
        moveStart: function() {
            this.start = this.pos;
        },
        makeSubStream: function(start, length, dict) {
            return new Stream(this.bytes.buffer, start, length, dict);
        }
    };

    return constructor;
})();

var StringStream = (function() {
    function constructor(str) {
        var length = str.length;
        var bytes = new Uint8Array(length);
        for (var n = 0; n < length; ++n)
            bytes[n] = str.charCodeAt(n);
        Stream.call(this, bytes);
    }

    constructor.prototype = Stream.prototype;

    return constructor;
})();

var FlateStream = (function() {
    const codeLenCodeMap = [
        16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
    ];

    const lengthDecode = [
        [0,   3],
        [0,   4],
        [0,   5],
        [0,   6],
        [0,   7],
        [0,   8],
        [0,   9],
        [0,  10],
        [1,  11],
        [1,  13],
        [1,  15],
        [1,  17],
        [2,  19],
        [2,  23],
        [2,  27],
        [2,  31],
        [3,  35],
        [3,  43],
        [3,  51],
        [3,  59],
        [4,  67],<commit_msg>make codeLenCodeMap a typed array<commit_after>/* -*- Mode: Java; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- /
/* vim: set shiftwidth=4 tabstop=8 autoindent cindent expandtab: */

var ERRORS = 0, WARNINGS = 1, TODOS = 5;
var verbosity = WARNINGS;

function log(msg) {
    if (console && console.log)
        console.log(msg);
    else if (print)
        print(msg);
}

function warn(msg) {
    if (verbosity >= WARNINGS)
        log("Warning: "+ msg);
}

function error(msg) {
    throw new Error(msg);
}

function TODO(what) {
    if (verbosity >= TODOS)
        log("TODO: "+ what);
}

function malformed(msg) {
    error("Malformed PDF: "+ msg);
}

function assert(cond, msg) {
    if (!cond)
        error(msg);
}

// In a well-formed PDF, |cond| holds.  If it doesn't, subsequent
// behavior is undefined.
function assertWellFormed(cond, msg) {
    if (!cond)
        malformed(msg);
}

function shadow(obj, prop, value) {
    Object.defineProperty(obj, prop, { value: value, enumerable: true });
    return value;
}

var Stream = (function() {
    function constructor(arrayBuffer, start, length, dict) {
        this.bytes = new Uint8Array(arrayBuffer);
        this.start = start || 0;
        this.pos = this.start;
        this.end = (start + length) || arrayBuffer.byteLength;
        this.dict = dict;
    }

    constructor.prototype = {
        get length() {
            return this.end - this.start;
        },
        getByte: function() {
            var bytes = this.bytes;
            if (this.pos >= this.end)
                return -1;
            return bytes[this.pos++];
        },
        lookChar: function() {
            var bytes = this.bytes;
            if (this.pos >= this.end)
                return;
            return String.fromCharCode(bytes[this.pos]);
        },
        getChar: function() {
            var ch = this.lookChar();
            if (!ch)
                return ch;
            this.pos++;
            return ch;
        },
        skip: function(n) {
            if (!n)
                n = 1;
            this.pos += n;
        },
        reset: function() {
            this.pos = this.start;
        },
        moveStart: function() {
            this.start = this.pos;
        },
        makeSubStream: function(start, length, dict) {
            return new Stream(this.bytes.buffer, start, length, dict);
        }
    };

    return constructor;
})();

var StringStream = (function() {
    function constructor(str) {
        var length = str.length;
        var bytes = new Uint8Array(length);
        for (var n = 0; n < length; ++n)
            bytes[n] = str.charCodeAt(n);
        Stream.call(this, bytes);
    }

    constructor.prototype = Stream.prototype;

    return constructor;
})();

var FlateStream = (function() {
    const codeLenCodeMap = new Uint32Array([
        16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
    ]);

    const lengthDecode = [
        [0,   3],
        [0,   4],
        [0,   5],
        [0,   6],
        [0,   7],
        [0,   8],
        [0,   9],
        [0,  10],
        [1,  11],
        [1,  13],
        [1,  15],
        [1,  17],
        [2,  19],
        [2,  23],
        [2,  27],
        [2,  31],
        [3,  35],
        [3,  43],
        [3,  51],
        [3,  59],
        [4,  67],
{'fraction_non_alphanumeric': 0.1482597559138165, 'fraction_numerical': 0.029531414795841496, 'mean_word_length': 1.6257911392405062, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4339116', 'n_tokens_mistral': 2262, 'n_tokens_neox': 2132, 'n_words': 571}