<commit_before>import { MuSchema } from './schema';
import { MuNumber } from './_number';
import {
    MuReadStream,
    MuWriteStream,
} from 'mustreams';

const numType2ArrayType = {
    float32: Float32Array,
    float64: Float64Array,
    int8: Int8Array,
    int16: Int16Array,
    int32: Int32Array,
    uint8: Uint8Array,
    uint16: Uint16Array,
    uint32: Uint32Array,
};

export type _MuVectorType<ValueSchema extends MuNumber> = {
    float32:Float32Array;
    float64:Float64Array;
    int8:Int8Array;
    int16:Int16Array;
    int32:Int32Array;
    uint8:Uint8Array;
    uint16:Uint16Array;
    uint32:Uint32Array;
}[ValueSchema['muType']];

export class MuVector<ValueSchema extends MuNumber> implements MuSchema<_MuVectorType<ValueSchema>> {
    private _constructor:typeof numType2ArrayType[ValueSchema['muType']];
    private _pool:_MuVectorType<ValueSchema>[] = [];

    public readonly identity:_MuVectorType<ValueSchema>;
    public readonly muType = 'vector';
    public readonly muData:ValueSchema;
    public readonly json:object;

    public readonly dimension:number;

    constructor (valueSchema:ValueSchema, dimension:number) {
        this._constructor = numType2ArrayType[valueSchema.muType];

        this.identity = new this._constructor(dimension);
        for (let i = 0; i < dimension; ++i) {
            this.identity[i] = valueSchema.identity;
        }

        this.muData = valueSchema;
        this.json = {
            type: 'vector',
            valueType: this.muData.json,
            dimension,
        };
        this.dimension = dimension;
    }

    public alloc () : _MuVectorType<ValueSchema> {
        return this._pool.pop() || new this._constructor(this.dimension);
    }

    public free (vec) {
        this._pool.push(vec);
    }

    public clone (vec) : _MuVectorType<ValueSchema> {
        const copy = this.alloc();
        copy.set(vec);
        return copy;
    }

    // TODO remove diff and patch
    public diff () {}
    public patch () { return new Uint8Array(0); }

    public diffBinary (base_:_MuVectorType<ValueSchema>, target_:_MuVectorType<ValueSchema>, stream:MuWriteStream) {
        const valueSchema:MuSchema<number> = this.muData;
        const dimension = this.dimension * this.identity.BYTES_PER_ELEMENT;

        stream.grow(Math.ceil(dimension * 9 / 8));

        const base = new Uint8Array(base_.buffer);
        const target = new Uint8Array(target_.buffer);

        let tracker = 0;
        let numPatch = 0;
        for (let i = 0; i < dimension; ++i) {
            if (base[i] !== target[i]) {
                tracker |= 1 << (i & 7);
                ++numPatch;<commit_msg>implement diff and patch for vector<commit_after>import { MuSchema } from './schema';
import { MuNumber } from './_number';
import {
    MuReadStream,
    MuWriteStream,
} from 'mustreams';

const numType2ArrayType = {
    float32: Float32Array,
    float64: Float64Array,
    int8: Int8Array,
    int16: Int16Array,
    int32: Int32Array,
    uint8: Uint8Array,
    uint16: Uint16Array,
    uint32: Uint32Array,
};

export type _MuVectorType<ValueSchema extends MuNumber> = {
    float32:Float32Array;
    float64:Float64Array;
    int8:Int8Array;
    int16:Int16Array;
    int32:Int32Array;
    uint8:Uint8Array;
    uint16:Uint16Array;
    uint32:Uint32Array;
}[ValueSchema['muType']];

export class MuVector<ValueSchema extends MuNumber> implements MuSchema<_MuVectorType<ValueSchema>> {
    private _constructor:typeof numType2ArrayType[ValueSchema['muType']];
    private _pool:_MuVectorType<ValueSchema>[] = [];

    public readonly identity:_MuVectorType<ValueSchema>;
    public readonly muType = 'vector';
    public readonly muData:ValueSchema;
    public readonly json:object;

    public readonly dimension:number;

    constructor (valueSchema:ValueSchema, dimension:number) {
        this._constructor = numType2ArrayType[valueSchema.muType];

        this.identity = new this._constructor(dimension);
        for (let i = 0; i < dimension; ++i) {
            this.identity[i] = valueSchema.identity;
        }

        this.muData = valueSchema;
        this.json = {
            type: 'vector',
            valueType: this.muData.json,
            dimension,
        };
        this.dimension = dimension;
    }

    public alloc () : _MuVectorType<ValueSchema> {
        return this._pool.pop() || new this._constructor(this.dimension);
    }

    public free (vec) {
        this._pool.push(vec);
    }

    public clone (vec) : _MuVectorType<ValueSchema> {
        const copy = this.alloc();
        copy.set(vec);
        return copy;
    }

    public diff (a:_MuVectorType<ValueSchema>, b:_MuVectorType<ValueSchema>) : (any|undefined) {
        for (let i = 0; i < a.length; ++i) {
            if (a[i] !== b[i]) {
                return Array.prototype.slice.call(b);
            }
        }
        return;
    }

    public patch (a:_MuVectorType<ValueSchema>, p:any) {
        if (p) {
            const x = this.alloc();
            for (let i = 0; i < p.length; ++i) {
                x[i] = p[i];
            }
            return x;
        }
        return this.clone(a);
    }

    public diffBinary (base_:_MuVectorType<ValueSchema>, target_:_MuVectorType<ValueSchema>, stream:MuWriteStream) {
        const valueSchema:MuSchema<number> = this.muData;
        const dimension = this.dimension * this.identity.BYTES_PER_ELEMENT;

        stream.grow(Math.ceil(dimension * 9 / 8));

        const base = new Uint8Array(base_.buffer);
        const target = new Uint8Array(target_.buffer);

        let tracker = 0;
        let numPatch = 0;
        for (let i = 0; i < dimension; ++i) {
            if (base[i] !== target[i]) {
                tracker |= 1 << (i & 7);
                ++numPatch;
{'fraction_non_alphanumeric': 0.11550414364640885, 'fraction_numerical': 0.02451657458563536, 'mean_word_length': 2.607098381070984, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1422226', 'n_tokens_mistral': 1848, 'n_tokens_neox': 1732, 'n_words': 386}