<commit_before>import * as fse from 'fs-extra';
import { inflateBuffer } from './inflate';

export class DictZipParser {

    // read 64KB each time
    private _readLen: number = 64 * 1024;

    private _header: DzHeader;
    private _fd: number

    constructor(fd: number) {
        this._fd = fd;
    }

    async parse(pos: number, len: number): Promise<Buffer> {
        console.log(`parsing file from pos ${pos}, len: ${len}`);
        if(this._header == undefined) {
            this._header = await this.getHeader();
        }
        console.log(this._header);
        let headerLength = this.getHeaderLen(this._header);

        let CHLEN: number = this._header.FEXTRA.FIELD.SUBFIELD.CHLEN.readUInt16LE(0);
        let CHCNT: number = this._header.FEXTRA.FIELD.SUBFIELD.CHCNT.readUInt16LE(0);
        let CHUNKS: Buffer = this._header.FEXTRA.FIELD.SUBFIELD.CHUNKS;

        let startChunkIndex = ~~(pos / CHLEN);
        let endChunkIndex = ~~((pos + len) / CHLEN);

        // startDecompressPos: headerLength + CHUNKS.readUInt16LE(0) + CHUNKS.readUInt16LE(1) + ... + CHUNKS.readUInt16LE(pos / CHLEN - 1)
        // endDecompressPos: headerLength + CHUNKS.readUInt16LE(0) + CHUNKS.readUInt16LE(1) + ... + CHUNKS.readUInt16LE((pos + len) / CHLEN)
        // include startDecompressPos, exclude endDecompressPos
        let startDecompressPos = headerLength;
        let endDecompressPos = headerLength;
        for(let i = 0; i <= ~~((pos + len) / CHLEN); i++) {
            if(CHUNKS.length <= i * 2) {
                // the specified length has surpassed the end of the
                // original file
                break;
             }
            if(i <= ~~(pos / CHLEN) - 1) {
                startDecompressPos += CHUNKS.readUInt16LE(i)
            }
            endDecompressPos += CHUNKS.readUInt16LE(i);
        }


        let compressedData: Buffer = Buffer.alloc(endDecompressPos - startDecompressPos);
        await fse.read(this._fd, compressedData, 0, compressedData.length, startDecompressPos);

        console.log(new Uint8Array(compressedData));
        let decompressedData = inflateBuffer(new Uint8Array(compressedData));
        return decompressedData.slice(pos - ~~(pos / CHLEN) * CHCNT, pos + len - ~~(pos / CHLEN) * CHCNT);
    }
    
    private async getHeader(): Promise<DzHeader> {
        let header = <DzHeader> {};
        
        // read the first 10 bytes
        let buffer = Buffer.alloc(10);
        await fse.read(this._fd, buffer, 0, buffer.length, 0);
        header.ID1 = buffer.slice(0, 1);
        header.ID2 = buffer.slice(1, 2);
        header.CM = buffer.slice(2, 3);
        header.FLG = buffer.slice(3, 4);
        header.MTIME = buffer.slice(4, 8);
        header.XFL = buffer.slice(8, 9);
        header.OS = buffer.slice(9, 10);
        if(header.ID1.readUInt8(0) != parseInt('0x1F', 16) || header.ID2.readUInt8(0) != parseInt('0x8B', 16)) {
            throw new Error(`Not a dictzip file`);
        }

        let pos = 10;

        // read FEXTRA if exists
        if((header.FLG.readUInt8(0) & 4) != 0) {
            // FEXTRA bit is set 1
            let XLEN = Buffer.alloc(2);
            await fse.read(this._fd, XLEN, 0, XLEN.length, pos);

            // read field in FEXTRA
            let FIELD = Buffer.alloc(XLEN.readUInt16LE(0));
            await fse.read(this._fd, FIELD, 0, FIELD.length, pos + 2);
            let SI1 = FIELD.slice(0, 1);
            let SI2 = FIELD.slice(1, 2);
            let LEN = FIELD.slice(2, 4);
            let SUBFIELD = FIELD.slice(4, 4 + LEN.readUInt16LE(0));
            let VER = SUBFIELD.slice(0, 2);
            let CHLEN = SUBFIELD.slice(2, 4);
            let CHCNT = SUBFIELD.slice(4, 6);
            let CHUNKS = SUBFIELD.slice(6, CHCNT.readUInt16LE(0) + 7);
            if(String.fromCharCode(SI1.readUInt8(0)) != 'R' || String.fromCharCode(SI2.readUInt8(0)) != 'A') {
                throw new Error("Not a dictzip File, SI1 or SI2 don't match.");
            }
            header.FEXTRA = {
                XLEN: XLEN,
                FIELD: {
                    SI1: SI1,
                    SI2: SI2,
                    LEN: LEN,
                    SUBFIELD: {
                        VER: VER,
                        CHLEN: CHLEN,
                        CHCNT: CHCNT,
                        CHUNKS: CHUNKS
                    }
                }
            };
            pos = pos + XLEN.readUInt16LE(0) + 2;
        } else {
            throw new Error("FEXTRA field in the dictzip header is not set, maybe not a dictzip file?");
        }
        // read FNAME
        if((header.FLG.readUInt8(0) & 8) != 0) {
            header.FNAME = await this.readToZero(this._fd, pos);
            pos += header.FNAME.length;<commit_msg>CHUNKS is measured by words instead of bytes<commit_after>import * as fse from 'fs-extra';
import { inflateBuffer } from './inflate';

export class DictZipParser {

    // read 64KB each time
    private _readLen: number = 64 * 1024;

    private _header: DzHeader;
    private _fd: number

    constructor(fd: number) {
        this._fd = fd;
    }

    async parse(pos: number, len: number): Promise<Buffer> {
        console.log(`parsing file from pos ${pos}, len: ${len}`);
        if(this._header == undefined) {
            this._header = await this.getHeader();
        }
        console.log(this._header);
        let headerLength = this.getHeaderLen(this._header);

        let CHLEN: number = this._header.FEXTRA.FIELD.SUBFIELD.CHLEN.readUInt16LE(0);
        let CHCNT: number = this._header.FEXTRA.FIELD.SUBFIELD.CHCNT.readUInt16LE(0);
        let CHUNKS: Buffer = this._header.FEXTRA.FIELD.SUBFIELD.CHUNKS;

        let startChunkIndex = ~~(pos / CHLEN);
        let endChunkIndex = ~~((pos + len) / CHLEN);

        // startDecompressPos: headerLength + CHUNKS.readUInt16LE(0) + CHUNKS.readUInt16LE(1) + ... + CHUNKS.readUInt16LE(pos / CHLEN - 1)
        // endDecompressPos: headerLength + CHUNKS.readUInt16LE(0) + CHUNKS.readUInt16LE(1) + ... + CHUNKS.readUInt16LE((pos + len) / CHLEN)
        // include startDecompressPos, exclude endDecompressPos
        let startDecompressPos = headerLength;
        let endDecompressPos = headerLength;
        for(let i = 0; i <= ~~((pos + len) / CHLEN); i++) {
            if(CHUNKS.length <= i * 2) {
                // the specified length has surpassed the end of the
                // original file
                break;
             }
            if(i <= ~~(pos / CHLEN) - 1) {
                startDecompressPos += CHUNKS.readUInt16LE(i)
            }
            endDecompressPos += CHUNKS.readUInt16LE(i);
        }


        let compressedData: Buffer = Buffer.alloc(endDecompressPos - startDecompressPos);
        await fse.read(this._fd, compressedData, 0, compressedData.length, startDecompressPos);

        let decompressedData = inflateBuffer(new Uint8Array(compressedData));
        return decompressedData.slice(pos - ~~(pos / CHLEN) * CHCNT, pos + len - ~~(pos / CHLEN) * CHCNT);
    }
    
    private async getHeader(): Promise<DzHeader> {
        let header = <DzHeader> {};
        
        // read the first 10 bytes
        let buffer = Buffer.alloc(10);
        await fse.read(this._fd, buffer, 0, buffer.length, 0);
        header.ID1 = buffer.slice(0, 1);
        header.ID2 = buffer.slice(1, 2);
        header.CM = buffer.slice(2, 3);
        header.FLG = buffer.slice(3, 4);
        header.MTIME = buffer.slice(4, 8);
        header.XFL = buffer.slice(8, 9);
        header.OS = buffer.slice(9, 10);
        if(header.ID1.readUInt8(0) != parseInt('0x1F', 16) || header.ID2.readUInt8(0) != parseInt('0x8B', 16)) {
            throw new Error(`Not a dictzip file`);
        }

        let pos = 10;

        // read FEXTRA if exists
        if((header.FLG.readUInt8(0) & 4) != 0) {
            // FEXTRA bit is set 1
            let XLEN = Buffer.alloc(2);
            await fse.read(this._fd, XLEN, 0, XLEN.length, pos);

            // read field in FEXTRA
            let FIELD = Buffer.alloc(XLEN.readUInt16LE(0));
            await fse.read(this._fd, FIELD, 0, FIELD.length, pos + 2);
            let SI1 = FIELD.slice(0, 1);
            let SI2 = FIELD.slice(1, 2);
            let LEN = FIELD.slice(2, 4);
            let SUBFIELD = FIELD.slice(4, 4 + LEN.readUInt16LE(0));
            let VER = SUBFIELD.slice(0, 2);
            let CHLEN = SUBFIELD.slice(2, 4);
            let CHCNT = SUBFIELD.slice(4, 6);
            let CHUNKS = SUBFIELD.slice(6, CHCNT.readUInt16LE(0) * 2 + 7);
            if(String.fromCharCode(SI1.readUInt8(0)) != 'R' || String.fromCharCode(SI2.readUInt8(0)) != 'A') {
                throw new Error("Not a dictzip File, SI1 or SI2 doesn't match.");
            }
            header.FEXTRA = {
                XLEN: XLEN,
                FIELD: {
                    SI1: SI1,
                    SI2: SI2,
                    LEN: LEN,
                    SUBFIELD: {
                        VER: VER,
                        CHLEN: CHLEN,
                        CHCNT: CHCNT,
                        CHUNKS: CHUNKS
                    }
                }
            };
            pos = pos + XLEN.readUInt16LE(0) + 2;
        } else {
            throw new Error("FEXTRA field in the dictzip header is not set, maybe not a dictzip file?");
        }
        // read FNAME
        if((header.FLG.readUInt8(0) & 8) != 0) {
            header.FNAME = await this.readToZero(this._fd, pos);
            pos += header.FNAME.length;
{'fraction_non_alphanumeric': 0.12723261189325488, 'fraction_numerical': 0.028577432233662534, 'mean_word_length': 1.7939536248899326, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1543311', 'n_tokens_mistral': 3191, 'n_tokens_neox': 2994, 'n_words': 701}