<commit_before>let src =
  let src = Logs.Src.create "capture" ~doc:"capture network traffic" in
  Logs.Src.set_level src (Some Logs.Debug);
  src

module Log = (val Logs.src_log src : Logs.LOG)

module Cstructs = struct

  type t = Cstruct.t list

  let pp_t ppf t =
    List.iter (fun t ->
      Format.fprintf ppf "[%d,%d](%d)" t.Cstruct.off t.Cstruct.len (Bigarray.Array1.dim t.Cstruct.buffer)
    ) t

  let len = List.fold_left (fun acc c -> Cstruct.len c + acc) 0

  let err fmt =
    let b = Buffer.create 20 in                         (* for thread safety. *)
    let ppf = Format.formatter_of_buffer b in
    let k ppf = Format.pp_print_flush ppf (); invalid_arg (Buffer.contents b) in
    Format.kfprintf k ppf fmt

  let rec shift t x =
    if x = 0 then t else match t with
    | [] -> err "Cstructs.shift %a %d" pp_t t x
    | y :: ys ->
      let y' = Cstruct.len y in
      if y' > x
      then Cstruct.shift y x :: ys
      else shift ys (x - y')

  (* Return a Cstruct.t representing (off, len) by either returning a reference
     or making a copy if the value is split across two fragments. Ideally this
     would return a string rather than a Cstruct.t for efficiency *)
  let get f t off len =
    let t' = shift t off in
    match t' with
    | x :: xs ->
      if Cstruct.len x >= len
      then Cstruct.sub x 0 len
      else begin
        failwith "not implemented"<commit_msg>Implement missing case in `Cstructs.get`<commit_after>let src =
  let src = Logs.Src.create "capture" ~doc:"capture network traffic" in
  Logs.Src.set_level src (Some Logs.Debug);
  src

module Log = (val Logs.src_log src : Logs.LOG)

module Cstructs = struct

  type t = Cstruct.t list

  let pp_t ppf t =
    List.iter (fun t ->
      Format.fprintf ppf "[%d,%d](%d)" t.Cstruct.off t.Cstruct.len (Bigarray.Array1.dim t.Cstruct.buffer)
    ) t

  let len = List.fold_left (fun acc c -> Cstruct.len c + acc) 0

  let err fmt =
    let b = Buffer.create 20 in                         (* for thread safety. *)
    let ppf = Format.formatter_of_buffer b in
    let k ppf = Format.pp_print_flush ppf (); invalid_arg (Buffer.contents b) in
    Format.kfprintf k ppf fmt

  let rec shift t x =
    if x = 0 then t else match t with
    | [] -> err "Cstructs.shift %a %d" pp_t t x
    | y :: ys ->
      let y' = Cstruct.len y in
      if y' > x
      then Cstruct.shift y x :: ys
      else shift ys (x - y')

  (* Return a Cstruct.t representing (off, len) by either returning a reference
     or making a copy if the value is split across two fragments. Ideally this
     would return a string rather than a Cstruct.t for efficiency *)
  let get f t off len =
    let t' = shift t off in
    match t' with
    | x :: xs ->
      (* Return a reference to the existing buffer *)
      if Cstruct.len x >= len
      then Cstruct.sub x 0 len
      else begin
        (* Copy into a fresh buffer *)
        let rec copy remaining frags =
          if Cstruct.len remaining > 0
          then match frags with
            | [] ->
              err "invalid bounds in Cstructs.%s %a off=%d len=%d" f pp_t t off len
            | x :: xs ->
              let to_copy = min (Cstruct.len x) (Cstruct.len remaining) in
              Cstruct.blit x 0 remaining 0 to_copy;
              (* either we've copied all of x, or we've filled the remaining buffer *)
              copy (Cstruct.shift remaining to_copy) xs in
        let result = Cstruct.create len in
        copy result (x :: xs);
        result
{'fraction_non_alphanumeric': 0.09416141957641672, 'fraction_numerical': 0.004293073840870063, 'mean_word_length': 2.095659875996457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2730688', 'n_tokens_mistral': 1157, 'n_tokens_neox': 1120, 'n_words': 460}