<commit_before>// @flow
"use strict";
export type Primitive = string | number | boolean | null;
export type JsonValue = Primitive | JsonMap | JsonValue[];
export type JsonMap = { [key: string]: JsonValue };

export type Key = Primitive;
export type KeyRangeTo = { from?: number, to: number };
export type KeyRangeLength = { from?: number, length: number };
export type KeyRange = KeyRangeTo | KeyRangeLength;
export type Path = Key[];
export type KeySet = Key | KeyRange | Array<Key | KeyRange>;
export type PathSet = KeySet[];

export type JsonGraph = { [key: string]: JsonGraphNode, $type?: empty };
export type JsonGraphNode = JsonGraph | JsonGraphLeaf;
export type JsonGraphLeaf =
  | JsonGraphAtom
  | JsonGraphError
  | JsonGraphRef
  | Primitive;

export type JsonGraphAtomDefined<T: JsonValue> = JsonGraphMetadata & {
  $type: "atom",
  value: T
};

export type JsonGraphAtomUndefined = JsonGraphMetadata & {
  $type: "atom",
  value?: empty
};

export type JsonGraphAtom =
  | JsonGraphAtomDefined<JsonValue>
  | JsonGraphAtomUndefined;

export type JsonGraphError = JsonGraphMetadata & {
  $type: "error",
  value: JsonValue
};
export type JsonGraphRef = JsonGraphMetadata & {
  $type: "ref",
  value: Path
};
<commit_msg>fix: JsonMap and JsonGraph types should not assume values are present.<commit_after>// @flow
"use strict";
export type Primitive = string | number | boolean | null;
export type JsonValue = Primitive | JsonMap | JsonValue[];
export type JsonMap = { [key: string]: JsonValue | void };

export type Key = Primitive;
export type KeyRangeTo = { from?: number, to: number };
export type KeyRangeLength = { from?: number, length: number };
export type KeyRange = KeyRangeTo | KeyRangeLength;
export type Path = Key[];
export type KeySet = Key | KeyRange | Array<Key | KeyRange>;
export type PathSet = KeySet[];

export type JsonGraph = { [key: string]: JsonGraphNode | void, $type?: empty };
export type JsonGraphNode = JsonGraph | JsonGraphLeaf;
export type JsonGraphLeaf =
  | JsonGraphAtom
  | JsonGraphError
  | JsonGraphRef
  | Primitive;

export type JsonGraphAtomDefined<T: JsonValue> = JsonGraphMetadata & {
  $type: "atom",
  value: T
};

export type JsonGraphAtomUndefined = JsonGraphMetadata & {
  $type: "atom",
  value?: empty
};

export type JsonGraphAtom =
  | JsonGraphAtomDefined<JsonValue>
  | JsonGraphAtomUndefined;

export type JsonGraphError = JsonGraphMetadata & {
  $type: "error",
  value: JsonValue
};
export type JsonGraphRef = JsonGraphMetadata & {
  $type: "ref",
  value: Path
};

{'fraction_non_alphanumeric': 0.11124260355029586, 'fraction_numerical': 0.0, 'mean_word_length': 4.84331797235023, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2676262', 'n_tokens_mistral': 770, 'n_tokens_neox': 775, 'n_words': 250}