<commit_before>(*
 * Copyright (c) 2013-2015 <NAME> <<EMAIL>>
 * Copyright (c) 2015 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *)

open Lwt.Infix
open Irmin

module Conf = struct

  let min_size =
    Irmin.Private.Conf.key ~doc:"Minimal chunk size" "min-size"
      Irmin.Private.Conf.int 4000

  let chunk_size =
    Irmin.Private.Conf.key ~doc:"Size of chunk" "size"
      Irmin.Private.Conf.int 4666

end

let chunk_size = Conf.chunk_size

let err_too_small ~min size =
  Printf.ksprintf invalid_arg
    "Chunks of %d bytes are too small. Size should at least be %d bytes."
    size min

let config ?(config=Irmin.Private.Conf.empty) ?size ?min_size () =
  let module C = Irmin.Private.Conf in
  let min_size = match min_size with
    | None   -> C.default Conf.min_size
    | Some v -> v
  in
  let size = match size with
    | None   -> C.default Conf.chunk_size
    | Some v -> if v < min_size then err_too_small ~min:min_size v else v
  in
  C.add (C.add config Conf.min_size min_size) Conf.chunk_size size

module Chunk (K: Irmin.Hash.S) = struct

  type t = Data | Indirect

  let l_type = 1
  let get_type x = Cstruct.get_uint8 x 0
  let set_type x v = Cstruct.set_uint8 x 0 v

  let l_len = 2
  let get_length x = Cstruct.LE.get_uint16 x l_type
  let set_length x v = Cstruct.LE.set_uint16 x l_type v

  let head_length = l_type + l_len
  let offset_payload = head_length

  let get_sub_key x pos =
    let offset = offset_payload + (pos * K.digest_size) in
    Cstruct.sub x offset K.digest_size

  let set_data src srcoff dst dstoff len =
    Cstruct.blit src srcoff dst (offset_payload + dstoff) len

  let extract_data src srcoff dst dstoff len =
    Cstruct.blit src (offset_payload + srcoff) dst dstoff len

  let type_of_int = function
    | Indirect -> 0
    | Data     -> 1

  let int_of_type = function
    | 0 -> Indirect
    | 1 -> Data
    | _ -> failwith "Unknow type"

  let type_of_chunk x = int_of_type (get_type x)

  let create typ chunk_size len  =
    let c = Cstruct.create chunk_size in
    Cstruct.memset c 0x00;
    set_type c (type_of_int typ);
    set_length c len;
    c

end

module AO (S:AO_MAKER) (K:Irmin.Hash.S) (V: Irmin.Contents.Conv)
: sig
    include Irmin.AO
    val v : Irmin.config -> t Lwt.t
  end
    with type key = S(K)(V).key and type value = S(K)(V).value
= struct

  module AO = S(K)(V)
  type key = AO.key
  type value = AO.value

  module Chunk = Chunk(K)

  type t = {
    db          : AO.t;             (* An hanlder to the underlying database. *)
    chunk_size  : int;                                 (* the size of chunks. *)
    max_children: int;     (* the maximum number of children a node can have. *)
    max_length  : int; (* the maximum lentgh (in bytes) of data stored in one
                          chunk. *)
  }

  let v_of_raw raw =
    match Irmin.Type.decode_cstruct V.t raw with
    |Ok va -> va
    |Error _ -> invalid_arg "v_of_raw"

  let raw_of_v v =
    Irmin.Type.encode_cstruct V.t v

  module Tree = struct

    let get_child t node pos =
      let key = K.of_raw (Chunk.get_sub_key node pos) in
      AO.find t.db key >>= function
      |Some v -> Lwt.return_some @@ raw_of_v v
      |None -> Lwt.return_none<commit_msg>hanlder -> handler<commit_after>(*
 * Copyright (c) 2013-2015 <NAME> <<EMAIL>>
 * Copyright (c) 2015 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *)

open Lwt.Infix
open Irmin

module Conf = struct

  let min_size =
    Irmin.Private.Conf.key ~doc:"Minimal chunk size" "min-size"
      Irmin.Private.Conf.int 4000

  let chunk_size =
    Irmin.Private.Conf.key ~doc:"Size of chunk" "size"
      Irmin.Private.Conf.int 4666

end

let chunk_size = Conf.chunk_size

let err_too_small ~min size =
  Printf.ksprintf invalid_arg
    "Chunks of %d bytes are too small. Size should at least be %d bytes."
    size min

let config ?(config=Irmin.Private.Conf.empty) ?size ?min_size () =
  let module C = Irmin.Private.Conf in
  let min_size = match min_size with
    | None   -> C.default Conf.min_size
    | Some v -> v
  in
  let size = match size with
    | None   -> C.default Conf.chunk_size
    | Some v -> if v < min_size then err_too_small ~min:min_size v else v
  in
  C.add (C.add config Conf.min_size min_size) Conf.chunk_size size

module Chunk (K: Irmin.Hash.S) = struct

  type t = Data | Indirect

  let l_type = 1
  let get_type x = Cstruct.get_uint8 x 0
  let set_type x v = Cstruct.set_uint8 x 0 v

  let l_len = 2
  let get_length x = Cstruct.LE.get_uint16 x l_type
  let set_length x v = Cstruct.LE.set_uint16 x l_type v

  let head_length = l_type + l_len
  let offset_payload = head_length

  let get_sub_key x pos =
    let offset = offset_payload + (pos * K.digest_size) in
    Cstruct.sub x offset K.digest_size

  let set_data src srcoff dst dstoff len =
    Cstruct.blit src srcoff dst (offset_payload + dstoff) len

  let extract_data src srcoff dst dstoff len =
    Cstruct.blit src (offset_payload + srcoff) dst dstoff len

  let type_of_int = function
    | Indirect -> 0
    | Data     -> 1

  let int_of_type = function
    | 0 -> Indirect
    | 1 -> Data
    | _ -> failwith "Unknow type"

  let type_of_chunk x = int_of_type (get_type x)

  let create typ chunk_size len  =
    let c = Cstruct.create chunk_size in
    Cstruct.memset c 0x00;
    set_type c (type_of_int typ);
    set_length c len;
    c

end

module AO (S:AO_MAKER) (K:Irmin.Hash.S) (V: Irmin.Contents.Conv)
: sig
    include Irmin.AO
    val v : Irmin.config -> t Lwt.t
  end
    with type key = S(K)(V).key and type value = S(K)(V).value
= struct

  module AO = S(K)(V)
  type key = AO.key
  type value = AO.value

  module Chunk = Chunk(K)

  type t = {
    db          : AO.t;             (* An handler to the underlying database. *)
    chunk_size  : int;                                 (* the size of chunks. *)
    max_children: int;     (* the maximum number of children a node can have. *)
    max_length  : int; (* the maximum lentgh (in bytes) of data stored in one
                          chunk. *)
  }

  let v_of_raw raw =
    match Irmin.Type.decode_cstruct V.t raw with
    |Ok va -> va
    |Error _ -> invalid_arg "v_of_raw"

  let raw_of_v v =
    Irmin.Type.encode_cstruct V.t v

  module Tree = struct

    let get_child t node pos =
      let key = K.of_raw (Chunk.get_sub_key node pos) in
      AO.find t.db key >>= function
      |Some v -> Lwt.return_some @@ raw_of_v v
      |None -> Lwt.return_none
{'fraction_non_alphanumeric': 0.08342900880214313, 'fraction_numerical': 0.009439979589233321, 'mean_word_length': 2.69811320754717, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3721014', 'n_tokens_mistral': 2775, 'n_tokens_neox': 2628, 'n_words': 962}