<commit_before>/** required keys of an object, not undefined */
type RequiredKeys<T> = {
  [K in keyof T]-?: undefined extends T[K] ? never : K
}[keyof T]

/** optional or undifined-able keys of an object */
type OptionalKeys<T> = {
  [K in keyof T]-?: undefined extends T[K] ? K : never
}[keyof T]

/** type is true if T is a union type */
type IsUnion_<T, U extends T = T> = false extends (
  T extends unknown ? ([U] extends [T] ? false : true) : never
)
  ? false
  : true
type IsUnion<T> = IsUnion_<T>

/** type is true if T is identically E */
type TypeEquality<T, E> = [T] extends [E] ? ([E] extends [T] ? true : false) : false

/** type is true if T or null is identically E or null*/
type NullTypeEquality<T, E> = TypeEquality<T | null, E | null>

/** gets only the string literals of a type or null if a type isn't a string literal */
type EnumString<T> = [T] extends [never]
  ? null
  : T extends string
  ? string extends T
    ? null
    : T
  : null

/** true if type is a union of string literals */
type IsEnum<T> = null extends EnumString<Exclude<T, null>> ? false : true

/** true only if all types are array types (not tuples) */
// NOTE relies on the fact that tuples don't have an index at 0.5, but arrays
// have an index at every number
type IsElements<T> = false extends IsUnion<T>
  ? [T] extends [readonly unknown[]]
    ? undefined extends T[0.5]
      ? false
      : true
    : false
  : false

/** true if the the type is a values type */
type IsValues<T> = false extends IsUnion<Exclude<T, null>>
  ? TypeEquality<keyof Exclude<T, null>, string>
  : false

/** true if the type is a properties type */
type IsProperties<T> = false extends IsUnion<Exclude<T, null>>
  ? null extends EnumString<keyof Exclude<T, null>>
    ? false
    : true
  : false

/** true if type is discriminator type */
type IsDiscriminator<T> = true extends IsUnion<Exclude<T, null>>
  ? null extends EnumString<keyof Exclude<T, null>>
    ? false
    : true
  : false

/** numeric strings */
type NumberType = "float32" | "float64" | "int8" | "uint8" | "int16" | "uint16" | "int32" | "uint32"

/** string strings */
type StringType = "string" | "timestamp"

/** actual schema */
export type JTDSchemaType<T, D extends Record<string, unknown> = Record<string, never>> = (
  | // refs - where null wasn't specified, must match exactly
  (null extends EnumString<keyof D>
      ? never
      :
          | ({[K in keyof D]: [T] extends [D[K]] ? {ref: K} : never}[keyof D] & {nullable?: false})
          // nulled refs - if ref is nullable and nullable is specified, then it can
          // match either null or non-null definitions
          | (null extends T
              ? {
                  [K in keyof D]: [Exclude<T, null>] extends [Exclude<D[K], null>]
                    ? {ref: K}
                    : never
                }[keyof D] & {nullable: true}
              : never))
  // empty - empty schemas also treat nullable differently in that it's now fully ignored
  | (unknown extends T ? {nullable?: boolean} : never)
  // all other types
  | ((// numbers - only accepts the type number
    true extends NullTypeEquality<T, number>
      ? {type: NumberType}
      : // booleans - accepts the type boolean
      true extends NullTypeEquality<T, boolean>
      ? {type: "boolean"}
      : // strings - only accepts the type string
      true extends NullTypeEquality<T, string>
      ? {type: StringType}
      : // strings - only accepts the type Date
      true extends NullTypeEquality<T, Date>
      ? {type: "timestamp"}
      : // enums - only accepts union of string literals
      // TODO we can't actually check that everything in the union was specified
      true extends IsEnum<T>
      ? {enum: EnumString<Exclude<T, null>>[]}
      : // arrays - only accepts arrays, could be array of unions to be resolved later
      true extends IsElements<Exclude<T, null>>
      ? T extends readonly (infer E)[]
        ? {
            elements: JTDSchemaType<E, D>
          }
        : never
      : // values
      true extends IsValues<T>
      ? T extends Record<string, infer V>
        ? {
            values: JTDSchemaType<V>
          }
        : never
      : // properties
      true extends IsProperties<T>
      ? ([RequiredKeys<Exclude<T, null>>] extends [never]
          ? {
              properties?: Record<string, never>
            }
          : {
              properties: {[K in RequiredKeys<T>]: JTDSchemaType<T[K], D>}
            }) &
          ([OptionalKeys<Exclude<T, null>>] extends [never]
            ? {
                optionalProperties?: Record<string, never>
              }
            : {
                optionalProperties: {
                  [K in OptionalKeys<T>]: JTDSchemaType<Exclude<T[K], undefined>, D>
                }
              }) & {
            additionalProperties?: boolean
          }
      : // discriminator
      true extends IsDiscriminator<T>
      ? {<commit_msg>simplify with IsRecord<commit_after>/** required keys of an object, not undefined */
type RequiredKeys<T> = {
  [K in keyof T]-?: undefined extends T[K] ? never : K
}[keyof T]

/** optional or undifined-able keys of an object */
type OptionalKeys<T> = {
  [K in keyof T]-?: undefined extends T[K] ? K : never
}[keyof T]

/** type is true if T is a union type */
type IsUnion_<T, U extends T = T> = false extends (
  T extends unknown ? ([U] extends [T] ? false : true) : never
)
  ? false
  : true
type IsUnion<T> = IsUnion_<T>

/** type is true if T is identically E */
type TypeEquality<T, E> = [T] extends [E] ? ([E] extends [T] ? true : false) : false

/** type is true if T or null is identically E or null*/
type NullTypeEquality<T, E> = TypeEquality<T | null, E | null>

/** gets only the string literals of a type or null if a type isn't a string literal */
type EnumString<T> = [T] extends [never]
  ? null
  : T extends string
  ? string extends T
    ? null
    : T
  : null

/** true if type is a union of string literals */
type IsEnum<T> = null extends EnumString<Exclude<T, null>> ? false : true

/** true only if all types are array types (not tuples) */
// NOTE relies on the fact that tuples don't have an index at 0.5, but arrays
// have an index at every number
type IsElements<T> = false extends IsUnion<T>
  ? [T] extends [readonly unknown[]]
    ? undefined extends T[0.5]
      ? false
      : true
    : false
  : false

/** true if the the type is a values type */
type IsValues<T> = false extends IsUnion<Exclude<T, null>>
  ? TypeEquality<keyof Exclude<T, null>, string>
  : false

/** true if type is a proeprties type and Union is false, or type is a discriminator type and Union is true */
type IsRecord<T, Union extends boolean> = Union extends IsUnion<Exclude<T, null>>
  ? null extends EnumString<keyof Exclude<T, null>>
    ? false
    : true
  : false

/** numeric strings */
type NumberType = "float32" | "float64" | "int8" | "uint8" | "int16" | "uint16" | "int32" | "uint32"

/** string strings */
type StringType = "string" | "timestamp"

/** actual schema */
export type JTDSchemaType<T, D extends Record<string, unknown> = Record<string, never>> = (
  | // refs - where null wasn't specified, must match exactly
  (null extends EnumString<keyof D>
      ? never
      :
          | ({[K in keyof D]: [T] extends [D[K]] ? {ref: K} : never}[keyof D] & {nullable?: false})
          // nulled refs - if ref is nullable and nullable is specified, then it can
          // match either null or non-null definitions
          | (null extends T
              ? {
                  [K in keyof D]: [Exclude<T, null>] extends [Exclude<D[K], null>]
                    ? {ref: K}
                    : never
                }[keyof D] & {nullable: true}
              : never))
  // empty - empty schemas also treat nullable differently in that it's now fully ignored
  | (unknown extends T ? {nullable?: boolean} : never)
  // all other types
  | ((// numbers - only accepts the type number
    true extends NullTypeEquality<T, number>
      ? {type: NumberType}
      : // booleans - accepts the type boolean
      true extends NullTypeEquality<T, boolean>
      ? {type: "boolean"}
      : // strings - only accepts the type string
      true extends NullTypeEquality<T, string>
      ? {type: StringType}
      : // strings - only accepts the type Date
      true extends NullTypeEquality<T, Date>
      ? {type: "timestamp"}
      : // enums - only accepts union of string literals
      // TODO we can't actually check that everything in the union was specified
      true extends IsEnum<T>
      ? {enum: EnumString<Exclude<T, null>>[]}
      : // arrays - only accepts arrays, could be array of unions to be resolved later
      true extends IsElements<Exclude<T, null>>
      ? T extends readonly (infer E)[]
        ? {
            elements: JTDSchemaType<E, D>
          }
        : never
      : // values
      true extends IsValues<T>
      ? T extends Record<string, infer V>
        ? {
            values: JTDSchemaType<V>
          }
        : never
      : // properties
      true extends IsRecord<T, false>
      ? ([RequiredKeys<Exclude<T, null>>] extends [never]
          ? {
              properties?: Record<string, never>
            }
          : {
              properties: {[K in RequiredKeys<T>]: JTDSchemaType<T[K], D>}
            }) &
          ([OptionalKeys<Exclude<T, null>>] extends [never]
            ? {
                optionalProperties?: Record<string, never>
              }
            : {
                optionalProperties: {
                  [K in OptionalKeys<T>]: JTDSchemaType<Exclude<T[K], undefined>, D>
                }
              }) & {
            additionalProperties?: boolean
          }
      : // discriminator
      true extends IsRecord<T, true>
      ? {
{'fraction_non_alphanumeric': 0.11991434689507495, 'fraction_numerical': 0.0036708473539308656, 'mean_word_length': 2.2769796191112595, 'pattern_counts': {'":': 0, '<': 120, '<?xml version=': 0, '>': 120, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5289266', 'n_tokens_mistral': 2984, 'n_tokens_neox': 2926, 'n_words': 1064}