<commit_before>"use strict";

import { OrderedSet, Set, Map, OrderedMap } from "immutable";
import * as pluralize from "pluralize";

import { Value, Tag, valueTag, CompressedJSON } from "./CompressedJSON";
import { Type, PrimitiveType, EnumType, MapType, ArrayType, ClassType, UnionType } from "./Type";
import { assertNever, assert } from "./Support";
import { TypeGraphBuilder, UnionBuilder, TypeRef } from "./TypeBuilder";
import { shouldBeMap } from "./InferMaps";

const MIN_LENGTH_FOR_ENUM = 10;

// This should be the recursive type
//   Value[] | NestedValueArray[]
// but TypeScript doesn't support that.
type NestedValueArray = any;

function forEachArrayInNestedValueArray(va: NestedValueArray, f: (va: Value[]) => void): void {
    if (va.length === 0) {
        return;
    }
    if (Array.isArray(va[0])) {
        for (const x of va) {
            forEachArrayInNestedValueArray(x, f);
        }
    } else {
        f(va);
    }
}

function forEachValueInNestedValueArray(va: NestedValueArray, f: (v: Value) => void): void {
    forEachArrayInNestedValueArray(va, a => a.forEach(f));
}

class InferenceUnionBuilder extends UnionBuilder<NestedValueArray, NestedValueArray, any> {
    constructor(
        typeBuilder: TypeGraphBuilder,
        typeName: string,
        isInferred: boolean,
        private readonly _typeInference: TypeInference,
        private readonly _cjson: CompressedJSON,
        private readonly _numValues: number
    ) {
        super(typeBuilder, typeName, isInferred);
    }

    protected makeEnum(enumCases: string[]): TypeRef | null {
        assert(enumCases.length > 0);
        if (enumCases.length < Math.sqrt(this._numValues)) {
            return this.typeBuilder.getEnumType(this.typeName, true, OrderedSet(enumCases));
        }
        return null;
    }

    protected makeClass(classes: NestedValueArray, maps: any[]): TypeRef {
        assert(maps.length === 0);
        return this._typeInference.inferClassType(this._cjson, this.typeName, this.isInferred, classes);
    }

    protected makeArray(arrays: NestedValueArray): TypeRef {
        return this.typeBuilder.getArrayType(
            this._typeInference.inferType(this._cjson, pluralize.singular(this.typeName), this.isInferred, arrays)
        );
    }
}

export class TypeInference {
    constructor(
        private readonly _typeBuilder: TypeGraphBuilder,
        private readonly _inferMaps: boolean,
        private readonly _inferEnums: boolean
    ) {}

    inferType = (
        cjson: CompressedJSON,
        typeName: string,
        isInferred: boolean,
        valueArray: NestedValueArray
    ): TypeRef => {
        const unionBuilder = new InferenceUnionBuilder(
            this._typeBuilder,
            typeName,
            isInferred,
            this,
            cjson,
            valueArray.length
        );

        forEachValueInNestedValueArray(valueArray, value => {
            const t = valueTag(value);
            switch (t) {
                case Tag.Null:
                    unionBuilder.addNull();
                    break;
                case Tag.False:
                case Tag.True:
                    unionBuilder.addBool();
                    break;
                case Tag.Integer:
                    unionBuilder.addInteger();
                    break;
                case Tag.Double:
                    unionBuilder.addDouble();
                    break;
                case Tag.InternedString:
                    if (this._inferEnums && !unionBuilder.haveString && valueArray.length >= MIN_LENGTH_FOR_ENUM) {
                        const s = cjson.getStringForValue(value);
                        unionBuilder.addEnumCase(s);
                    } else {
                        unionBuilder.addStringType("string");
                    }
                    break;
                case Tag.UninternedString:
                    unionBuilder.addStringType("string");
                    break;
                case Tag.Object:
                    unionBuilder.addClass(cjson.getObjectForValue(value));
                    break;
                case Tag.Array:
                    unionBuilder.addArray(cjson.getArrayForValue(value));
                    break;<commit_msg>Don't allow date/time strings to be enum types<commit_after>"use strict";

import { OrderedSet, Set, Map, OrderedMap } from "immutable";
import * as pluralize from "pluralize";

import { Value, Tag, valueTag, CompressedJSON } from "./CompressedJSON";
import { Type, PrimitiveType, EnumType, MapType, ArrayType, ClassType, UnionType } from "./Type";
import { assertNever, assert } from "./Support";
import { TypeGraphBuilder, UnionBuilder, TypeRef } from "./TypeBuilder";
import { shouldBeMap } from "./InferMaps";
import { isTime, isDateTime, isDate } from "./DateTime";

const MIN_LENGTH_FOR_ENUM = 10;

// This should be the recursive type
//   Value[] | NestedValueArray[]
// but TypeScript doesn't support that.
type NestedValueArray = any;

function forEachArrayInNestedValueArray(va: NestedValueArray, f: (va: Value[]) => void): void {
    if (va.length === 0) {
        return;
    }
    if (Array.isArray(va[0])) {
        for (const x of va) {
            forEachArrayInNestedValueArray(x, f);
        }
    } else {
        f(va);
    }
}

function forEachValueInNestedValueArray(va: NestedValueArray, f: (v: Value) => void): void {
    forEachArrayInNestedValueArray(va, a => a.forEach(f));
}

class InferenceUnionBuilder extends UnionBuilder<NestedValueArray, NestedValueArray, any> {
    constructor(
        typeBuilder: TypeGraphBuilder,
        typeName: string,
        isInferred: boolean,
        private readonly _typeInference: TypeInference,
        private readonly _cjson: CompressedJSON,
        private readonly _numValues: number
    ) {
        super(typeBuilder, typeName, isInferred);
    }

    protected makeEnum(enumCases: string[]): TypeRef | null {
        assert(enumCases.length > 0);
        if (enumCases.length < Math.sqrt(this._numValues)) {
            return this.typeBuilder.getEnumType(this.typeName, true, OrderedSet(enumCases));
        }
        return null;
    }

    protected makeClass(classes: NestedValueArray, maps: any[]): TypeRef {
        assert(maps.length === 0);
        return this._typeInference.inferClassType(this._cjson, this.typeName, this.isInferred, classes);
    }

    protected makeArray(arrays: NestedValueArray): TypeRef {
        return this.typeBuilder.getArrayType(
            this._typeInference.inferType(this._cjson, pluralize.singular(this.typeName), this.isInferred, arrays)
        );
    }
}

function canBeEnumCase(s: string): boolean {
    if (s.length === 0) return true; // FIXME: Do we really want this?
    if ("0123456789".indexOf(s[0]) < 0) return false;
    return !isDate(s) && !isTime(s, false) && !isDateTime(s);
}

export class TypeInference {
    constructor(
        private readonly _typeBuilder: TypeGraphBuilder,
        private readonly _inferMaps: boolean,
        private readonly _inferEnums: boolean
    ) {}

    inferType = (
        cjson: CompressedJSON,
        typeName: string,
        isInferred: boolean,
        valueArray: NestedValueArray
    ): TypeRef => {
        const unionBuilder = new InferenceUnionBuilder(
            this._typeBuilder,
            typeName,
            isInferred,
            this,
            cjson,
            valueArray.length
        );

        forEachValueInNestedValueArray(valueArray, value => {
            const t = valueTag(value);
            switch (t) {
                case Tag.Null:
                    unionBuilder.addNull();
                    break;
                case Tag.False:
                case Tag.True:
                    unionBuilder.addBool();
                    break;
                case Tag.Integer:
                    unionBuilder.addInteger();
                    break;
                case Tag.Double:
                    unionBuilder.addDouble();
                    break;
                case Tag.InternedString:
                    if (this._inferEnums && !unionBuilder.haveString && valueArray.length >= MIN_LENGTH_FOR_ENUM) {
                        const s = cjson.getStringForValue(value);
                        if (canBeEnumCase(s)) {
                            unionBuilder.addEnumCase(s);
                        } else {
                            unionBuilder.addStringType("string");
                        }
                    } else {
                        unionBuilder.addStringType("string");
                    }
                    break;
                case Tag.UninternedString:
                    unionBuilder.addStringType("string");
                    break;
                case Tag.Object:
                    unionBuilder.addClass(cjson.getObjectForValue(value));
                    break;
                case Tag.Array:
                    unionBuilder.addArray(cjson.getArrayForValue(value));
                    break;
{'fraction_non_alphanumeric': 0.09702486678507993, 'fraction_numerical': 0.002775310834813499, 'mean_word_length': 1.9722863741339491, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6564960', 'n_tokens_mistral': 2471, 'n_tokens_neox': 2374, 'n_words': 547}