<commit_before>open Ketrew_pure.Internal_pervasives
open Unix_io

module Ring = struct
  type 'a t = {
    recent: 'a option array;
    mutable index: int;

  }
  let create ?(size = 4200) () =
    {recent = Array.make size None; index = 0}

  let add t v =
    t.recent.(t.index) <- Some v;
    begin match (t.index + 1) with
    | i when i = (Array.length t.recent) -> t.index <- 0
    | other -> t.index <- (t.index + 1);
    end;
    ()

  let fold t ~init ~f =
    let rec go count prev index =
      let next item = f prev item in
      match index, t.recent.(index) with
      | _, None -> prev
      | _, _ when count = Array.length t.recent -> prev
      | 0, Some s ->
        go (count + 1) (next s) (Array.length t.recent - 1)
      | n, Some s ->
        go (count + 1) (next s) (n - 1)
    in
    if t.index = 0 then go 0 init (Array.length t.recent - 1) else go 0 init (t.index - 1) 

  let clear t =
    t.index <- 0;
    Array.iteri t.recent  ~f:(fun i _ -> t.recent.(i) <- None)

end

module Log_store = struct
  type t = Typed_log.Item.t Ring.t

  let create ?size () : t =
    Ring.create ?size ()


  let add t l = Ring.add t l

  let append_to_file ~path ~format t =
    let as_list = Ring.fold ~init:[] t ~f:(fun prev x -> x :: prev) in
    IO.with_out_channel (`Append_to_file path) ~f:(fun out ->
        IO.write out (if format = `Json then "[\n" else "")
        >>= fun () ->
        List.fold ~init:(return true) as_list
          ~f:(fun prev_m item ->
              prev_m >>= fun prev ->
              IO.write out
                (if format = `Json then
                   (if prev then "\n" else ",\n")
                 else "\n" ^ String.make 80 '-' ^ "\n")
              >>= fun () ->
              IO.write out
                (match format with
                | `Json ->
                  Yojson.Safe.pretty_to_string (Typed_log.Item.to_yojson item)
                | `Txt ->
                  Typed_log.Item.show item)
              >>= fun () ->
              return false
            )
        >>= fun (_ : bool) ->
        IO.write out (if format = `Json then "\n]\n" else "\n")
      )

  let clear t =
    Ring.clear t;
    return ()

end

module Global = struct

  let _log = Log_store.create ()

  module Logger = struct
    include Typed_log.Item.Construct
    include Display_markup
    let add l =
      Log_store.add _log l
    let log l = add (now l)

  end
  module Make_module_error_and_info(M: sig val module_name : string end) = struct
    let log_error e lo =
      let open Logger in
      let open Typed_log.Item in
      description_list [
        Constants.word_module, text M.module_name;
        Constants.word_error, text (Error.to_string e);
        Constants.word_info, text (Log.to_long_string lo);
      ] |> log
    let log_info lo =
      let open Typed_log.Item in
      let open Logger in
      description_list [
        Constants.word_module, text M.module_name;
        Constants.word_info, text (Log.to_long_string lo);
      ] |> log

  end

  let append_to_file ~path ~format = Log_store.append_to_file ~path ~format _log
  let clear () = Log_store.clear _log

end
<commit_msg>Refactor the Ring ds in Logging<commit_after>open Ketrew_pure.Internal_pervasives
open Unix_io

(* A parametric circular buffer. *)
module Ring = struct
  type 'a t = {
    recent: 'a option array;
    mutable index: int;
  }

  let create ?(size = 4200) () =
    {recent = Array.make size None; index = 0}

  let add t v =
    t.recent.(t.index) <- Some v;
    t.index <- (t.index + 1) mod (Array.length t.recent)

  let fold_right t ~init ~f =
    let l = Array.length t.recent in
    let p i = (i - 1 + l) mod l in
    let rec go count prev index =
      let next item = f prev item in
      match index, t.recent.(index) with
      | _, None -> prev
      | _, _ when count = l -> prev
      | n, Some s ->
        go (count + 1) (next s) (p n)
    in
    go 0 init (p t.index)

  let clear t =
    t.index <- 0;
    Array.iteri t.recent  ~f:(fun i _ -> t.recent.(i) <- None)

end (* Ring *)

module Log_store = struct
  type t = Typed_log.Item.t Ring.t

  let create ?size () : t =
    Ring.create ?size ()

  let add t l = Ring.add t l

  let append_to_file ~path ~format t =
    let as_list = Ring.fold_right ~init:[] t ~f:(fun prev x -> x :: prev) in
    IO.with_out_channel (`Append_to_file path) ~f:(fun out ->
        IO.write out (if format = `Json then "[\n" else "")
        >>= fun () ->
        List.fold ~init:(return true) as_list
          ~f:(fun prev_m item ->
              prev_m >>= fun prev ->
              IO.write out
                (if format = `Json then
                   (if prev then "\n" else ",\n")
                 else "\n" ^ String.make 80 '-' ^ "\n")
              >>= fun () ->
              IO.write out
                (match format with
                | `Json ->
                  Yojson.Safe.pretty_to_string (Typed_log.Item.to_yojson item)
                | `Txt ->
                  Typed_log.Item.show item)
              >>= fun () ->
              return false
            )
        >>= fun (_ : bool) ->
        IO.write out (if format = `Json then "\n]\n" else "\n")
      )

  let clear t =
    Ring.clear t;
    return ()

end (* Log_store *)

module Global = struct

  let _log = Log_store.create ()

  module Logger = struct
    include Typed_log.Item.Construct
    include Display_markup
    let add l =
      Log_store.add _log l
    let log l = add (now l)

  end
  module Make_module_error_and_info(M: sig val module_name : string end) = struct
    let log_error e lo =
      let open Logger in
      let open Typed_log.Item in
      description_list [
        Constants.word_module, text M.module_name;
        Constants.word_error, text (Error.to_string e);
        Constants.word_info, text (Log.to_long_string lo);
      ] |> log
    let log_info lo =
      let open Logger in
      let open Typed_log.Item in
      description_list [
        Constants.word_module, text M.module_name;
        Constants.word_info, text (Log.to_long_string lo);
      ] |> log

  end

  let append_to_file ~path ~format = Log_store.append_to_file ~path ~format _log
  let clear () = Log_store.clear _log

end (* Global *)

{'fraction_non_alphanumeric': 0.11596530677802763, 'fraction_numerical': 0.00530035335689046, 'mean_word_length': 1.8024302430243024, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1830922', 'n_tokens_mistral': 2158, 'n_tokens_neox': 2028, 'n_words': 660}