<commit_before>open Batteries

type index

type t = {
  index : index;
  mutable closed : bool;
}

type vt =
  | Double of float
  | Long of int
  | String of string

type kv = {
  k : string;
  v : vt;
}

exception Invalid_index

external create : string -> index = "ml_grib_index_new"
external from_file : string -> string -> index = "ml_grib_index_new_from_file"
external add_file : index -> string -> unit = "ml_grib_index_add_file"

external delete : index -> unit = "ml_grib_index_delete"

let delete ({ index; closed } as t) =
  if closed then (
    (* Nothing to do *)
  )
  else (
    delete index;
    t.closed <- true;
  )

let init index =
  let t = { index; closed = false } in
  Gc.finalise delete t;
  t

let create keys =
  let keys = String.concat "," keys in
  init (create keys)

let of_file filename keys =
  let keys = String.concat "," keys in
  init (from_file filename keys)

let use { index; closed } f =
  if closed then raise Invalid_index;
  f index

let add_file index filename =
  use index (fun i -> add_file i filename)

external select_double : index -> string -> float -> unit = "ml_grib_index_select_double"
external select_long : index -> string -> int -> unit = "ml_grib_index_select_long"
external select_string : index -> string -> string -> unit = "ml_grib_index_select_string"

external size : index -> string -> int = "ml_grib_index_get_size"

external next_handle : index -> Handle.t option = "ml_grib_handle_new_from_index"
let next_handle index =
  use index next_handle<commit_msg>Wrap handles properly when iterating over an index<commit_after>open Batteries

type index

type t = {
  index : index;
  mutable closed : bool;
}

type vt =
  | Double of float
  | Long of int
  | String of string

type kv = {
  k : string;
  v : vt;
}

exception Invalid_index

external create : string -> index = "ml_grib_index_new"
external from_file : string -> string -> index = "ml_grib_index_new_from_file"
external add_file : index -> string -> unit = "ml_grib_index_add_file"

external delete : index -> unit = "ml_grib_index_delete"

let delete ({ index; closed } as t) =
  if closed then (
    (* Nothing to do *)
  )
  else (
    delete index;
    t.closed <- true;
  )

let init index =
  let t = { index; closed = false } in
  Gc.finalise delete t;
  t

let create keys =
  let keys = String.concat "," keys in
  init (create keys)

let of_file filename keys =
  let keys = String.concat "," keys in
  init (from_file filename keys)

let use { index; closed } f =
  if closed then raise Invalid_index;
  f index

let add_file index filename =
  use index (fun i -> add_file i filename)

external select_double : index -> string -> float -> unit = "ml_grib_index_select_double"
external select_long : index -> string -> int -> unit = "ml_grib_index_select_long"
external select_string : index -> string -> string -> unit = "ml_grib_index_select_string"

external size : index -> string -> int = "ml_grib_index_get_size"

external next_handle : index -> Handle.handle option = "ml_grib_handle_new_from_index"
let next_handle index =
  use index (fun i -> Handle.init (next_handle i))
{'fraction_non_alphanumeric': 0.09702209414024976, 'fraction_numerical': 0.0, 'mean_word_length': 3.4820659971305594, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1591138', 'n_tokens_mistral': 1029, 'n_tokens_neox': 1026, 'n_words': 346}