<commit_before>
import { pair } from './parser_combinators'
import { TextStream } from './string_combinators'

export { StringCharStream }

class StringCharStream implements TextStream {
    protected constructor(
        private source: string,
        private offset: number,
        private lineOffsetTable: [number, string][]
    ) {
    }

    protected newInstance(source: string, offset: number, lineOffsetTable: [number, string][]): this {
        return new StringCharStream(source, offset, lineOffsetTable) as this;
    }

    public static newInstance(source: string) {
        return new StringCharStream(source, 0, parseLineOffsets(source));
    }

    next() {
        if (this.offset >= this.source.length) {
            return null;
        }

        return pair(
            this.source[this.offset],
            this.newInstance(this.source, this.offset + 1, this.lineOffsetTable)
        );
    }

    getPosition() {
        return this.offset;
    }

    getLineCol(offset: number): [number, number] {
        let idx = binarySearch(this.lineOffsetTable, x => x[0] < offset ? -1 :
                                                          x[0] > offset ?  1 : 0);

        if (idx === false) {
            throw new Error(`Invalid offset: ${offset}`);
        }
        if (idx < 0) {
            idx = -idx - 1;
        }

        return [idx, offset - this.lineOffsetTable[idx][0]];
    }
}

function parseLineOffsets(source: string) {
    const lines = source.split('\n');

    let acc = [];
    let offset = 0;
    for (const l of lines) {
        acc.push(pair(offset, l));
        offset += l.length + 1;
    }

    return acc;
}

function binarySearch<T>(arr: T[], compare: (x: T) => -1|0|1): false|number {
    let low = 0;
    let high = arr.length - 1;

    if (!arr.length) {
        return false;
    }

    while (low <= high) {
        const m = low + ((high - low) >> 1);
        const cmp = compare(arr[m]);

        if (cmp < 0) {
            low = m + 1;
            continue;
        }

        if (cmp > 0) {
            high = m - 1;
            continue;
        }

        return m;
    }

    return low;
}<commit_msg>Fixed a bug in the 'not found' case of the bsearch implementation<commit_after>
import { pair } from './parser_combinators'
import { TextStream } from './string_combinators'

export { StringCharStream }

class StringCharStream implements TextStream {
    protected constructor(
        private source: string,
        private offset: number,
        private lineOffsetTable: [number, string][]
    ) {
    }

    protected newInstance(source: string, offset: number, lineOffsetTable: [number, string][]): this {
        return new StringCharStream(source, offset, lineOffsetTable) as this;
    }

    public static newInstance(source: string) {
        return new StringCharStream(source, 0, parseLineOffsets(source));
    }

    next() {
        if (this.offset >= this.source.length) {
            return null;
        }

        return pair(
            this.source[this.offset],
            this.newInstance(this.source, this.offset + 1, this.lineOffsetTable)
        );
    }

    getPosition() {
        return this.offset;
    }

    getLineCol(offset: number): [number, number] {
        let idx = binarySearch(this.lineOffsetTable, x => x[0] < offset ? -1 :
                                                          x[0] > offset ?  1 : 0);

        if (idx === false) {
            throw new Error(`Invalid offset: ${offset}`);
        }
        if (idx < 0) {
            idx = -idx - 1;
        }

        return [idx, offset - this.lineOffsetTable[idx][0]];
    }
}

function parseLineOffsets(source: string) {
    const lines = source.split('\n');

    let acc = [];
    let offset = 0;
    for (const l of lines) {
        acc.push(pair(offset, l));
        offset += l.length + 1;
    }

    return acc;
}

function binarySearch<T>(arr: T[], compare: (x: T) => -1|0|1): false|number {
    let low = 0;
    let high = arr.length - 1;

    if (!arr.length) {
        return false;
    }

    while (low <= high) {
        const m = low + ((high - low) >> 1);
        const cmp = compare(arr[m]);

        if (cmp < 0) {
            low = m + 1;
            continue;
        }

        if (cmp > 0) {
            high = m - 1;
            continue;
        }

        return m;
    }

    return -low;
}
{'fraction_non_alphanumeric': 0.1271399223921479, 'fraction_numerical': 0.010043369093814198, 'mean_word_length': 1.785759694850604, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7216960', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1278, 'n_words': 357}