<commit_before>import Long from 'long';
import { Buffer } from 'buffer';
import { createMessage } from './utils';
import { escape, isUndefined } from 'lodash';

class Deserializer {
    constructor(buffer, offset = 0) {
        this.buffer = buffer;
        this.offset = offset;
    }

    onError(...args) {
        throw new Error(createMessage.apply(null, args));
    }

    readUInt() {
        const bytes = this._readBytes(4);

        return bytes.readUInt32LE(0);
    }

    readInt() {
        const bytes = this._readBytes(4);

        return bytes.readInt32LE(0);
    }

    readShort() {
        const bytes = this._readBytes(2);

        return bytes.readInt16LE(0);
    }

    readUShort() {
        const bytes = this._readBytes(2);

        return bytes.readUInt16LE(0);
    }

    readFloat() {
        const bytes = this._readBytes(4);

        return bytes.readFloatLE(0);
    }

    readDouble() {
        const bytes = this._readBytes(8);

        return bytes.readDoubleLE(0);
    }

    readBool() {
        const bytes = this._readBytes(1);
        const bool = bytes.readUInt8(0);

        if(bool === 1) {
            return true;
        }

        if(bool === 0) {
            return false;
        }

        this.onError('Unexpected %s value for bool. Valid inputs are: 0 or 1', bool);
    }

    readString() {
        return this._readString(this.readBytes());
    }

    _readString(bytes) {
        let sUTF8 = '';
        const length = bytes.byteLength;

        for(let i = 0; i < length; i++) {
            sUTF8 += String.fromCodePoint(bytes[i]);
        }

        return decodeURIComponent(escape(sUTF8));
    }

    _readBytes(byteLength) {
        if(isUndefined(byteLength)) {
            this.onError('"%s" argument is mandatory', 'byteLength');
            return false;
        }

        const end = this.offset + byteLength;

        if(end > this.buffer.length) {
            this.onError('Byte length (value: %s) is larger than the given buffer, aborting (buffer length: %s, measured end: %s)',
                byteLength,
                this.buffer.length,
                end);
        }

        const buffer = this.buffer.slice(this.offset, end);
        this.offset = end;

        return buffer;
    }

    readBytes() {
        const length = this.readUInt();
        const bytes = this._readBytes(length);

        this.offset += length % 4;

        return bytes;
    }

    readLong() {
        const bytes = this._readBytes(8);

        const low = bytes.readInt32LE(0);
        const high = bytes.readInt32LE(4);

        const long = new Long(low, high, false);
        return '0x' + Buffer.from(long.toBytesBE()).toString('hex');
    }

    readULong() {<commit_msg>Deserializer: store byte length in a variable to avoid accessing it multiple times<commit_after>import Long from 'long';
import { Buffer } from 'buffer';
import { createMessage } from './utils';
import { escape, isUndefined } from 'lodash';

class Deserializer {
    constructor(buffer, offset = 0) {
        this.buffer = buffer;
        this.offset = offset;
    }

    onError(...args) {
        throw new Error(createMessage.apply(null, args));
    }

    readUInt() {
        const bytes = this._readBytes(4);

        return bytes.readUInt32LE(0);
    }

    readInt() {
        const bytes = this._readBytes(4);

        return bytes.readInt32LE(0);
    }

    readShort() {
        const bytes = this._readBytes(2);

        return bytes.readInt16LE(0);
    }

    readUShort() {
        const bytes = this._readBytes(2);

        return bytes.readUInt16LE(0);
    }

    readFloat() {
        const bytes = this._readBytes(4);

        return bytes.readFloatLE(0);
    }

    readDouble() {
        const bytes = this._readBytes(8);

        return bytes.readDoubleLE(0);
    }

    readBool() {
        const bytes = this._readBytes(1);
        const bool = bytes.readUInt8(0);

        if(bool === 1) {
            return true;
        }

        if(bool === 0) {
            return false;
        }

        this.onError('Unexpected %s value for bool. Valid inputs are: 0 or 1', bool);
    }

    readString() {
        return this._readString(this.readBytes());
    }

    _readString(bytes) {
        let sUTF8 = '';
        const length = bytes.byteLength;

        for(let i = 0; i < length; i++) {
            sUTF8 += String.fromCodePoint(bytes[i]);
        }

        return decodeURIComponent(escape(sUTF8));
    }

    _readBytes(byteLength) {
        if(isUndefined(byteLength)) {
            this.onError('"%s" argument is mandatory', 'byteLength');
            return false;
        }

        const end = this.offset + byteLength;
        const length = this.buffer.byteLength;

        if(end > length) {
            this.onError(
                'Byte length (value: %s) ', byteLength,

                'is larger than the given buffer, ',
                'aborting (buffer length: %s, ', length,
                'measured end: %s)', end);
            return false;
        }

        const buffer = this.buffer.slice(this.offset, end);
        this.offset = end;

        return buffer;
    }

    readBytes() {
        const length = this.readUInt();
        const bytes = this._readBytes(length);

        this.offset += length % 4;

        return bytes;
    }

    readLong() {
        const bytes = this._readBytes(8);

        const low = bytes.readInt32LE(0);
        const high = bytes.readInt32LE(4);

        const long = new Long(low, high, false);
        return '0x' + Buffer.from(long.toBytesBE()).toString('hex');
    }

    readULong() {
{'fraction_non_alphanumeric': 0.12522265764161025, 'fraction_numerical': 0.01460634128963306, 'mean_word_length': 1.8854059609455294, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '725389', 'n_tokens_mistral': 1741, 'n_tokens_neox': 1642, 'n_words': 394}