<commit_before>(* Tools used by the generated OCaml code *)
open Batteries
open Stdint
open RamenLog
open Lwt

(* Converters from string to values *)

let float_of_string = Pervasives.float_of_string
let string_of_string x = x
let bool_of_string = Pervasives.bool_of_string
let u8_of_string = Uint8.of_string
let u16_of_string = Uint16.of_string
let u32_of_string = Uint32.of_string
let u64_of_string = Uint64.of_string
let u128_of_string = Uint128.of_string
let i8_of_string = Int8.of_string
let i16_of_string = Int16.of_string
let i32_of_string = Int32.of_string
let i64_of_string = Int64.of_string
let i128_of_string = Int128.of_string
let null_of_string = ()
let eth_of_string s = Uint48.of_string ("0x"^ String.nreplace s ":" "")
let ip4_of_string = Ipv4.of_string
let ip6_of_string = Ipv6.of_string
let cidr4_of_string = Ipv4.Cidr.of_string
let cidr6_of_string = Ipv6.Cidr.of_string

(* Functions *)

(* We are not allowed to have any state specific to this function.
 * Consequently we must compute the sequence number from the start
 * and increment and the global tuple count. *)
(* FIXME: that's fine but now we do have internal state for functions.
 * And we want sequence(start,step) to be reset at start at every
 * group (and maybe another, stateless, global sequence). *)
let sequence start inc =
  Int128.(start + (of_uint64 !CodeGenLib_IO.tuple_count) * inc)

let age_float x = x -. !CodeGenLib_IO.now
let age_u8 = Uint8.of_float % age_float
let age_u16 = Uint16.of_float % age_float
let age_u32 = Uint32.of_float % age_float
let age_u64 = Uint64.of_float % age_float
let age_u128 = Uint128.of_float % age_float
let age_i8 = Int8.of_float % age_float
let age_i16 = Int16.of_float % age_float
let age_i32 = Int32.of_float % age_float
let age_i64 = Int64.of_float % age_float
let age_i128 = Int128.of_float % age_float
(* FIXME: typecheck age_eth, age_ipv4 etc out of existence *)

let now () = !CodeGenLib_IO.now

let percentile_init x = [x]
let percentile_add prev x = x::prev
let percentile_finalize pct lst =
  let arr = Array.of_list lst in
  Array.fast_sort Pervasives.compare arr ;
  assert (pct >= 0.0 && pct <= 100.0) ;
  let pct = pct *. 0.01 in
  let idx = Helpers.round_to_int (pct *. float_of_int (Array.length arr - 1)) in
  arr.(idx)

let smooth prev alpha x = x *. alpha +. prev *. (1. -. alpha)

let split by what k =
  String.nsplit ~by what |> Lwt_list.iter_s k

(* Remember values *)
type remember_state =
  { filter : RamenBloomFilter.sliced_filter ;
    mutable last_remembered : bool }

let remember_init tim dur e =
  let nb_slices = 10 and false_positive_ratio = 0.15 in<commit_msg>Fix bloom filter false positive ratio<commit_after>(* Tools used by the generated OCaml code *)
open Batteries
open Stdint
open RamenLog
open Lwt

(* Converters from string to values *)

let float_of_string = Pervasives.float_of_string
let string_of_string x = x
let bool_of_string = Pervasives.bool_of_string
let u8_of_string = Uint8.of_string
let u16_of_string = Uint16.of_string
let u32_of_string = Uint32.of_string
let u64_of_string = Uint64.of_string
let u128_of_string = Uint128.of_string
let i8_of_string = Int8.of_string
let i16_of_string = Int16.of_string
let i32_of_string = Int32.of_string
let i64_of_string = Int64.of_string
let i128_of_string = Int128.of_string
let null_of_string = ()
let eth_of_string s = Uint48.of_string ("0x"^ String.nreplace s ":" "")
let ip4_of_string = Ipv4.of_string
let ip6_of_string = Ipv6.of_string
let cidr4_of_string = Ipv4.Cidr.of_string
let cidr6_of_string = Ipv6.Cidr.of_string

(* Functions *)

(* We are not allowed to have any state specific to this function.
 * Consequently we must compute the sequence number from the start
 * and increment and the global tuple count. *)
(* FIXME: that's fine but now we do have internal state for functions.
 * And we want sequence(start,step) to be reset at start at every
 * group (and maybe another, stateless, global sequence). *)
let sequence start inc =
  Int128.(start + (of_uint64 !CodeGenLib_IO.tuple_count) * inc)

let age_float x = x -. !CodeGenLib_IO.now
let age_u8 = Uint8.of_float % age_float
let age_u16 = Uint16.of_float % age_float
let age_u32 = Uint32.of_float % age_float
let age_u64 = Uint64.of_float % age_float
let age_u128 = Uint128.of_float % age_float
let age_i8 = Int8.of_float % age_float
let age_i16 = Int16.of_float % age_float
let age_i32 = Int32.of_float % age_float
let age_i64 = Int64.of_float % age_float
let age_i128 = Int128.of_float % age_float
(* FIXME: typecheck age_eth, age_ipv4 etc out of existence *)

let now () = !CodeGenLib_IO.now

let percentile_init x = [x]
let percentile_add prev x = x::prev
let percentile_finalize pct lst =
  let arr = Array.of_list lst in
  Array.fast_sort Pervasives.compare arr ;
  assert (pct >= 0.0 && pct <= 100.0) ;
  let pct = pct *. 0.01 in
  let idx = Helpers.round_to_int (pct *. float_of_int (Array.length arr - 1)) in
  arr.(idx)

let smooth prev alpha x = x *. alpha +. prev *. (1. -. alpha)

let split by what k =
  String.nsplit ~by what |> Lwt_list.iter_s k

(* Remember values *)
type remember_state =
  { filter : RamenBloomFilter.sliced_filter ;
    mutable last_remembered : bool }

let remember_init tim dur e =
  let nb_slices = 10 and false_positive_ratio = 0.015 in
{'fraction_non_alphanumeric': 0.08350730688935282, 'fraction_numerical': 0.043082178781552474, 'mean_word_length': 4.4106776180698155, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1601959', 'n_tokens_mistral': 2082, 'n_tokens_neox': 1949, 'n_words': 580}