<commit_before>import * as utils from "./utils";

export interface SchemaDDL {
    [key: string]: TableDDL;
}

export interface FieldDDL {
    type?: FieldType,
    constraint?: ConstraintType;
    references?: string;
    relationName?: string;
}

export interface TableDDL {
    [key: string]: FieldDDL;
}

const PK = "PK", FK = "FK", NONE = "NONE";

export type ConstraintType = "PK" | "FK" | "NONE";
export type FieldType = "string" | "number" | "any";

export interface DatabaseSchema {
    name: string;
    tables: TableSchema[];
}

export interface DatabaseState {
    [key: string]: TableState;
}

export interface TableState {
    byId: { [key: string]: any };
    ids: string[];
}

export interface RecordState {
    id: string;
    state: any;
}

export interface Schema {
    name: string;

    getPrimaryKey: (state: any) => string;
}

export interface Session {
    name: string;
    state: DatabaseState;
}

export interface NormalizedState {
    [key: string]: { // schema name
        ids: string[],
        byId: {
            [key: string]: any
        }
    };
}

export class TableSchema {
    readonly name: string;
    readonly fields: FieldSchema[];

    relations: FieldSchema[] = [];
    private _primaryKeyFields: string[];
    private _foreignKeyFields: string[];

    constructor(name: string, schema: TableDDL) {
        this.name = name;
        this.fields = Object.keys(schema.fields).map(fieldName => new FieldSchema(this, fieldName, schema[fieldName]));

        this._primaryKeyFields = this.fields.filter(f => f.constraint === PK).map(f => f.name);
        this._foreignKeyFields = this.fields.filter(f => f.constraint === FK).map(f => f.name);
    }

    connect(schemas: TableSchema[]) {
        schemas.forEach(schema => {
            this.relations = this.relations.concat(schema.fields.filter(f => f.references === this.name));
        })
    }

    normalize(data: any, output: NormalizedState = {}) {
        if (!utils.isPlainObject(data) && !Array.isArray(data))
            throw new Error("Failed to normalize data. Given argument is not a plain object nor an array.");

        if (output[this.name])
            throw new Error("Failed to normalize data. Circular reference detected.");

        output[this.name] = { ids: [], byId: {} };
        output[this.name].ids = utils.ensureArray(data).map(obj => {
            const pk = this.getPrimaryKey(data);
            output[this.name].byId[pk] = obj;

            const relations: Record<string, any> = {};
            this.relations.forEach(rel => {
                if (rel.relationName && data[rel.relationName]) {
                    rel.table.normalize(data[rel.relationName], output);
                    delete data[rel.relationName];
                }
            });

            return pk;
        });

        return output;
    }

    getPrimaryKey(record: any) {
        const lookup = (this._primaryKeyFields.length ? this._primaryKeyFields : this._foreignKeyFields);
        const pk = lookup.reduce((p, n) => {
            const k = record[n];
            return p && k ? (p + "_" + k) : k;
        });

        if (!pk || pk.length === 0)
            throw new Error(`Failed to get primary key for record of type \"${this.name}\".`);

        return pk;
    }
}

export class FieldSchema {
    readonly table: TableSchema;
    readonly name: string;

    readonly type: FieldType;
    readonly constraint: ConstraintType;

    readonly references?: string;
    readonly relationName?: string;

    constructor(table: TableSchema, name: string, schema: FieldDDL) {
        this.table = table;
        this.name = name;
        this.type = schema.type || "any";
        this.constraint = schema.constraint || "NONE";
        this.references = schema.references;
        this.relationName = schema.relationName;
    }
}<commit_msg>TableSchema bugfix<commit_after>import * as utils from "./utils";

export interface SchemaDDL {
    [key: string]: TableDDL;
}

export interface FieldDDL {
    type?: FieldType,
    constraint?: ConstraintType;
    references?: string;
    relationName?: string;
}

export interface TableDDL {
    [key: string]: FieldDDL;
}

const PK = "PK", FK = "FK", NONE = "NONE";

export type ConstraintType = "PK" | "FK" | "NONE";
export type FieldType = "string" | "number" | "any";

export interface DatabaseSchema {
    name: string;
    tables: TableSchema[];
}

export interface DatabaseState {
    [key: string]: TableState;
}

export interface TableState {
    byId: { [key: string]: any };
    ids: string[];
}

export interface RecordState {
    id: string;
    state: any;
}

export interface Schema {
    name: string;

    getPrimaryKey: (state: any) => string;
}

export interface Session {
    name: string;
    state: DatabaseState;
}

export interface NormalizedState {
    [key: string]: { // schema name
        ids: string[],
        byId: {
            [key: string]: any
        }
    };
}

export class TableSchema {
    readonly name: string;
    readonly fields: FieldSchema[];

    relations: FieldSchema[] = [];
    private _primaryKeyFields: string[];
    private _foreignKeyFields: string[];

    constructor(name: string, schema: TableDDL) {
        this.name = name;
        this.fields = Object.keys(schema).map(fieldName => new FieldSchema(this, fieldName, schema[fieldName]));

        this._primaryKeyFields = this.fields.filter(f => f.constraint === PK).map(f => f.name);
        this._foreignKeyFields = this.fields.filter(f => f.constraint === FK).map(f => f.name);
    }

    connect(schemas: TableSchema[]) {
        schemas.forEach(schema => {
            this.relations = this.relations.concat(schema.fields.filter(f => f.references === this.name));
        })
    }

    normalize(data: any, output: NormalizedState = {}) {
        if (!utils.isPlainObject(data) && !Array.isArray(data))
            throw new Error("Failed to normalize data. Given argument is not a plain object nor an array.");

        if (output[this.name])
            throw new Error("Failed to normalize data. Circular reference detected.");

        output[this.name] = { ids: [], byId: {} };
        output[this.name].ids = utils.ensureArray(data).map(obj => {
            const pk = this.getPrimaryKey(data);
            output[this.name].byId[pk] = obj;

            const relations: Record<string, any> = {};
            this.relations.forEach(rel => {
                if (rel.relationName && data[rel.relationName]) {
                    rel.table.normalize(data[rel.relationName], output);
                    delete data[rel.relationName];
                }
            });

            return pk;
        });

        return output;
    }

    getPrimaryKey(record: any) {
        const lookup = (this._primaryKeyFields.length ? this._primaryKeyFields : this._foreignKeyFields);
        const pk = lookup.reduce((p, n) => {
            const k = record[n];
            return p && k ? (p + "_" + k) : k;
        });

        if (!pk || pk.length === 0)
            throw new Error(`Failed to get primary key for record of type \"${this.name}\".`);

        return pk;
    }
}

export class FieldSchema {
    readonly table: TableSchema;
    readonly name: string;

    readonly type: FieldType;
    readonly constraint: ConstraintType;

    readonly references?: string;
    readonly relationName?: string;

    constructor(table: TableSchema, name: string, schema: FieldDDL) {
        this.table = table;
        this.name = name;
        this.type = schema.type || "any";
        this.constraint = schema.constraint || "NONE";
        this.references = schema.references;
        this.relationName = schema.relationName;
    }
}
{'fraction_non_alphanumeric': 0.1327353094123765, 'fraction_numerical': 0.00026001040041601667, 'mean_word_length': 2.479421076436002, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2793746', 'n_tokens_mistral': 2242, 'n_tokens_neox': 2192, 'n_words': 581}