<commit_before>open Batteries;; open Printf

exception Error of string

type t = int

let to_int t = t

let to_probability t =
  10.0 ** (float_of_int t /. -10.0)

let int_of_offset = function `offset33 -> 33 | `offset64 -> 64

let to_ascii ?(offset=`offset33) t =
  let offset = int_of_offset offset in
  let x = t + offset in
  if offset <= x && x <= 126 then
    Char.chr x
  else
    Error (sprintf "%d with offset %d cannot be encoded as a visible ASCII character" t offset) |> raise

let of_int x =
  if x >= 0 then x
  else Error (sprintf "invalid PHRED score %d" x) |> raise

let of_ascii ?(offset=`offset33) x =
  let offset = int_of_offset offset in
  let c = Char.code x in
  if offset <= c && c <= 126 then
    c - offset
  else
    Error (sprintf "%c with offset %d is not a valid score" x offset) |> raise

let of_probability
    ?(f = fun x ->
      if mod_float x 1.0 < 0.5
      then int_of_float (floor x)
      else int_of_float (ceil x)
    )
    x =
  if 0.0 <= x && x <= 1.0 then
    f (-10. *. log10 x)
  else
    Error (sprintf "invalid probability %0.17g" x) |> raise
<commit_msg>[PhredScore] Factored out helper function for rounding floats to ints.<commit_after>open Batteries;; open Printf

exception Error of string

type t = int

let round_float_to_int x =
  if mod_float x 1.0 < 0.5
  then int_of_float (floor x)
  else int_of_float (ceil x)

let to_int t = t

let to_probability t =
  10.0 ** (float_of_int t /. -10.0)

let int_of_offset = function `offset33 -> 33 | `offset64 -> 64

let to_ascii ?(offset=`offset33) t =
  let offset = int_of_offset offset in
  let x = t + offset in
  if offset <= x && x <= 126 then
    Char.chr x
  else
    Error (sprintf "%d with offset %d cannot be encoded as a visible ASCII character" t offset) |> raise

let of_int x =
  if x >= 0 then x
  else Error (sprintf "invalid PHRED score %d" x) |> raise

let of_ascii ?(offset=`offset33) x =
  let offset = int_of_offset offset in
  let c = Char.code x in
  if offset <= c && c <= 126 then
    c - offset
  else
    Error (sprintf "%c with offset %d is not a valid score" x offset) |> raise

let of_probability ?(f = round_float_to_int) x =
  if 0.0 <= x && x <= 1.0 then
    f (-10. *. log10 x)
  else
    Error (sprintf "invalid probability %0.17g" x) |> raise

{'fraction_non_alphanumeric': 0.10526315789473684, 'fraction_numerical': 0.03508771929824561, 'mean_word_length': 2.8016666666666667, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5144009', 'n_tokens_mistral': 877, 'n_tokens_neox': 818, 'n_words': 333}