<commit_before>
include Cstruct

let append cs1 cs2 =
  let (l1, l2) = Cstruct.(len cs1, len cs2) in
  let cs = Cstruct.create (l1 + l2) in
  Cstruct.blit cs1 0 cs 0 l1;
  Cstruct.blit cs2 0 cs l1 l2;
  cs

let concat css =
  let result =
    Cstruct.create @@
      List.fold_left
        (fun a cs -> a + Cstruct.len cs)
        0 css in
  let _ =
    List.fold_left
      (fun off cs ->
        let n = Cstruct.len cs in
        Cstruct.blit cs 0 result off n ; off + n )
      0 css in
  result

let copy cs =
  let len = Cstruct.len cs in
  let cs' = Cstruct.create len in
  ( Cstruct.blit cs 0 cs' 0 len ; cs' )

let xor_into src dst =
  for i = 0 to Cstruct.len dst - 1 do
    Cstruct.(set_uint8 dst i (get_uint8 src i lxor get_uint8 dst i))
  done

let xor cs1 cs2 =
  let len = Cstruct.(min (len cs1) (len cs2)) in
  let cs = Cstruct.create len in
  for i = 0 to len - 1 do
    Cstruct.(set_uint8 cs i (get_uint8 cs1 i lxor get_uint8 cs2 i))
  done;
  cs

let fill cs x =
  for i = 0 to Cstruct.len cs - 1 do
    Cstruct.set_uint8 cs i x
  done

let create_with n x =
  let cs = Cstruct.create n in ( fill cs x ; cs )
<commit_msg>cs concat fast-path<commit_after>
include Cstruct

let append cs1 cs2 =
  let (l1, l2) = Cstruct.(len cs1, len cs2) in
  let cs = Cstruct.create (l1 + l2) in
  Cstruct.blit cs1 0 cs 0 l1;
  Cstruct.blit cs2 0 cs l1 l2;
  cs

let concat = function
  | []   -> Cstruct.create 0
  | [cs] -> cs
  | css  ->
      let result =
        Cstruct.create @@
          List.fold_left
            (fun a cs -> a + Cstruct.len cs)
            0 css in
      let _ =
        List.fold_left
          (fun off cs ->
            let n = Cstruct.len cs in
            Cstruct.blit cs 0 result off n ; off + n )
          0 css
      in
      result

let copy cs =
  let len = Cstruct.len cs in
  let cs' = Cstruct.create len in
  ( Cstruct.blit cs 0 cs' 0 len ; cs' )

let xor_into src dst =
  for i = 0 to Cstruct.len dst - 1 do
    Cstruct.(set_uint8 dst i (get_uint8 src i lxor get_uint8 dst i))
  done

let xor cs1 cs2 =
  let len = Cstruct.(min (len cs1) (len cs2)) in
  let cs = Cstruct.create len in
  for i = 0 to len - 1 do
    Cstruct.(set_uint8 cs i (get_uint8 cs1 i lxor get_uint8 cs2 i))
  done;
  cs

let fill cs x =
  for i = 0 to Cstruct.len cs - 1 do
    Cstruct.set_uint8 cs i x
  done

let create_with n x =
  let cs = Cstruct.create n in ( fill cs x ; cs )

{'fraction_non_alphanumeric': 0.0854522717799083, 'fraction_numerical': 0.033764068361817424, 'mean_word_length': 1.9593094944512948, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7342551', 'n_tokens_mistral': 966, 'n_tokens_neox': 934, 'n_words': 369}