<commit_before>'use strict';

const StringDecoder = require("string_decoder").StringDecoder;

/**
 * Makes an array of buffers appear as one long array of bytes
 */

class BufferContainer {

    constructor(buffers, args = []) {
        this.buffers = [];
        this.start = args.start || 0;
        this.end = args.end || 999999;
        this.pointer = this.start; // current position read
        this.bytesBefore = 0; // number of bytes from buffers that are removed from bufferContainer

        if( buffers instanceof BufferContainer ) {
            buffers = buffers.buffers;
        }

        // make sure buffers is an array
        buffers = [].concat( buffers );

        // add buffers
        buffers.forEach((buffer) => {
            this.push(buffer);
        });
    }

    resetBytesBefore() {
        this.bytesBefore = 0;
    }

    /**
     * @return {int} Total length of all buffers combined
     */
    length() {
        return this.bytesBefore + this.buffers.reduce((total, buffer) => { return total + buffer.length }, 0);
    }

    // add buffer chunk at the end
    push(buffer) {
        if( !Buffer.isBuffer(buffer) ) {
            throw new Error('Buffer should be of type buffer');
        }

        this.buffers.push(buffer);
    }

    // remove first element
    shift() {
        if( this.buffers.length ) {
            this.bytesBefore += this.buffers[0].length;

            // if pointer was on first buffer, set pointer to first byte of next buffer
            if( (this.bytesBefore + this.buffers[0].length) <= this.pointer ) {
                this.pointer = this.bytesBefore;
            }

            return this.buffers.shift();
        }
        return null;
    }

    each(cb, offset = 0) {
        offset += this.bytesBefore;
        let index = this.bytesBefore;

        for( let i = 0; i < this.buffers.length; i++ ) {
            let buffer = this.buffers[i];
            let bufferLength = buffer.length;

            if( offset > (index + bufferLength) ) {

                index += bufferLength;

            } else {

                for( let j = 0; j < bufferLength; j++ ) {
                    let byte = buffer[j];

                    if( cb(byte, index) === false ) {
                        return;
                    }

                    index += 1;
                }

            }
        }

    }

    eachGenerator() {
        let _this = this;

        return function* byteGenerator() {
            for( let i = 0; i < _this.buffers.length; i++ ) {
                let buffer = _this.buffers[i];

                for (var j = 0; j < buffer.length; j++) {
                    yield buffer[j];
                }
            }
        };
    }

    reduce(cb, initial = 0, offset = 0) {
        offset += this.bytesBefore;
        let index = this.bytesBefore;

        for( let i = 0; i < this.buffers.length; i++ ) {
            let buffer = this.buffers[i];
            let bufferLength = buffer.length;

            if( offset > (index + bufferLength) ) {

                index += bufferLength;

            } else {

                for (var j = 0; j < bufferLength; j++) {
                    initial = cb(initial, buffer[j], index);

                    index += 1;
                }

            }
        }

        return initial;
    }



    indexOf(searchByte, offset = 0) {
        let foundIndex = -1;

        this.each((byte, index) => {
            if( searchByte === byte ) {
                foundIndex = index;
                return false;
            }
        }, offset);

        return foundIndex;
    }

    indexOfAll(searchByte, offset = 0) {
        return this.reduce((arr, byte, index) => {
            if( searchByte === byte ) {
                arr.push(index);
            }
            return arr;
        }, [], offset);
    }

    byteAtIndex(searchIndex) {
        let b = null;

        this.each((byte, index) => {
            if( index === searchIndex ) {
                b = byte;
                return false;
            }
        }, searchIndex - 1);

        return b;
    }

    /**
     * Start inclusive
     * End exclusive
     * 
     * @param  {Number} start [description]
     * @param  {Number} end   [description]
     * @return {[type]}       [description]
     */
    createFrom(start = 0, end = 999999) {
        let container = new BufferContainer([]);
        let startIndex = 0;
        let endIndex = 999999;
        let originalStart = null; // index of start of first selected buffer

        for( let i = 0; i < this.buffers.length; i++ ) {

            let buffer = this.buffers[i];
            let bufferLength = buffer.length;

            endIndex = startIndex + bufferLength;
            // console.log("bufferLength", bufferLength, 'startIndex', startIndex, 'endIndex', endIndex);

            if( start < endIndex && end > startIndex) { // check good
                container.push(buffer);

                // set start offset 
                if( originalStart === null ) originalStart = startIndex;

                if( startIndex < start && start < endIndex ) {
                    container.start = start - startIndex;
                }

                if( endIndex > end && end > startIndex ) {
                    // console.log(`endindex(${endIndex}) is greater than end(${end}) and end(${end}) is greater than startIndex(${startIndex})`, ":", end, originalStart, end - originalStart);
                    container.end = end - originalStart;
                }
            }

            startIndex = endIndex;
        }

        // console.log("container offsets", "start", container.start, 'end', container.end);
<commit_msg>Added indexing to buffercontainer<commit_after>'use strict';

const StringDecoder = require("string_decoder").StringDecoder;

/**
 * Makes an array of buffers appear as one long array of bytes
 */

class BufferContainer {

    constructor(buffers, args = []) {
        this.buffers = [];
        this.start = args.start || 0;
        this.end = args.end || 999999;
        this.pointer = this.start; // current position read

        this._before = { // number of bytes or buffers that are removed from bufferContainer
            bytes: 0,
            buffers: 0
        };

        this._bufferIndex = {
            index: new Map(),
            last: null,           // index of last byte in map
            lastBufferIndex: null // index of last buffer in this.buffers
        };

        if( buffers instanceof BufferContainer ) {
            buffers = buffers.buffers;
        }

        // make sure buffers is an array
        buffers = [].concat( buffers );

        // add buffers
        buffers.forEach((buffer) => {
            this.push(buffer);
        });
    }

    indexBuffers() {
        let last = this._bufferIndex.last || -1;
        let lastBuffer = this._bufferIndex.lastBufferIndex || -1;
        const bufferLength = this.buffers.length + this._before.buffers;

        if( lastBuffer < bufferLength ) {
            for( let i = (lastBuffer + 1); i < bufferLength; i++ ) {
                lastBuffer = i;

                for( let j = 0; j < this.buffers[ (i - this._before.buffers) ].length; j++ ) {
                    last += 1;
                    this._bufferIndex.index.set(last, [i, j]);
                }

            }
        }
    }

    byteAtIndexed(index) {
        const i = this._bufferIndex.index.get(index);
        return this._buffers[ this._before.buffers + i[0] ][ i[1] ];
    }

    resetBytesBefore() {
        this._before.bytes = 0;
    }

    /**
     * @return {int} Total length of all buffers combined
     */
    length() {
        return this._before.bytes + this.buffers.reduce((total, buffer) => { return total + buffer.length; }, 0);
    }

    // add buffer chunk at the end
    push(buffer) {
        if( !Buffer.isBuffer(buffer) ) {
            throw new Error('Buffer should be of type buffer');
        }

        this.buffers.push(buffer);
        this.indexBuffers();
    }

    // remove first element
    shift() {
        if( this.buffers.length ) {
            this._before.bytes += this.buffers[0].length;
            this._before.buffers += 1;

            // if pointer was on first buffer, set pointer to first byte of next buffer
            if( (this._before.bytes + this.buffers[0].length) <= this.pointer ) {
                this.pointer = this._before.bytes;
            }

            // reindex buffersIndex


            return this.buffers.shift();
        }
        return null;
    }

    each(cb, offset = 0) {
        offset += this._before.bytes;
        let index = this._before.bytes;

        for( let i = 0; i < this.buffers.length; i++ ) {
            let buffer = this.buffers[i];
            let bufferLength = buffer.length;

            if( offset > (index + bufferLength) ) {

                index += bufferLength;

            } else {

                for( let j = 0; j < bufferLength; j++ ) {
                    let byte = buffer[j];

                    if( cb(byte, index) === false ) {
                        return;
                    }

                    index += 1;
                }

            }
        }

    }

    eachGenerator() {
        let _this = this;

        return function* byteGenerator() {
            for( let i = 0; i < _this.buffers.length; i++ ) {
                let buffer = _this.buffers[i];

                for (var j = 0; j < buffer.length; j++) {
                    yield buffer[j];
                }
            }
        };
    }

    reduce(cb, initial = 0, offset = 0) {
        offset += this._before.bytes;
        let index = this._before.bytes;

        for( let i = 0; i < this.buffers.length; i++ ) {
            let buffer = this.buffers[i];
            let bufferLength = buffer.length;

            if( offset > (index + bufferLength) ) {

                index += bufferLength;

            } else {

                for (var j = 0; j < bufferLength; j++) {
                    initial = cb(initial, buffer[j], index);

                    index += 1;
                }

            }
        }

        return initial;
    }



    indexOf(searchByte, offset = 0) {
        let foundIndex = -1;

        this.each((byte, index) => {
            if( searchByte === byte ) {
                foundIndex = index;
                return false;
            }
        }, offset);

        return foundIndex;
    }

    indexOfAll(searchByte, offset = 0) {
        return this.reduce((arr, byte, index) => {
            if( searchByte === byte ) {
                arr.push(index);
            }
            return arr;
        }, [], offset);
    }

    byteAtIndex(searchIndex) {
        let b = null;

        this.each((byte, index) => {
            if( index === searchIndex ) {
                b = byte;
                return false;
            }
        }, searchIndex - 1);

        return b;
    }

    /**
     * Start inclusive
     * End exclusive
     *
     * @param  {Number} start [description]
     * @param  {Number} end   [description]
     * @return {[type]}       [description]
     */
    createFrom(start = 0, end = 999999) {
        let container = new BufferContainer([]);
        let startIndex = 0;
        let endIndex = 999999;
        let originalStart = null; // index of start of first selected buffer

        for( let i = 0; i < this.buffers.length; i++ ) {

            let buffer = this.buffers[i];
            let bufferLength = buffer.length;

            endIndex = startIndex + bufferLength;
            // console.log("bufferLength", bufferLength, 'startIndex', startIndex, 'endIndex', endIndex);

            if( start < endIndex && end > startIndex) { // check good
                container.push(buffer);

                // set start offset
                if( originalStart === null ) originalStart = startIndex;

                if( startIndex < start && start < endIndex ) {
                    container.start = start - startIndex;
                }

                if( endIndex > end && end > startIndex ) {
                    // console.log(`endindex(${endIndex}) is greater than end(${end}) and end(${end}) is greater than startIndex(${startIndex})`, ":", end, originalStart, end - originalStart);
                    container.end = end - originalStart;
                }
            }

            startIndex = endIndex;
        }

        // console.log("container offsets", "start", container.start, 'end', container.end);

{'fraction_non_alphanumeric': 0.11870905895510465, 'fraction_numerical': 0.007429301805400224, 'mean_word_length': 1.4205336426914152, 'pattern_counts': {'":': 2, '<': 28, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5082197', 'n_tokens_mistral': 3547, 'n_tokens_neox': 3282, 'n_words': 911}