<commit_before>module UnlabeledList = List
module List = StdLabels.List
module String = StdLabels.String

let invalid_argf = Core_printf.invalid_argf

module T = struct
  type 'a t = 'a list
end

include T

let of_list t = t

let range ?(stride=1) ?(start=`inclusive) ?(stop=`exclusive) start_i stop_i =
  if stride = 0 then
    invalid_arg "Core_list.range: stride must be non-zero";
  (* Generate the range from the last element, so that we do not need to rev it *)
  let rec loop last counter accum =
    if counter <= 0 then accum
    else loop (last - stride) (counter - 1) (last :: accum)
  in
  let stride_sign = if stride > 0 then 1 else -1 in
  let start =
    match start with
    | `inclusive -> start_i
    | `exclusive -> start_i + stride
  in
  let stop =
    match stop with
    | `inclusive -> stop_i + stride_sign
    | `exclusive -> stop_i
  in
  let num_elts = (stop - start + stride - stride_sign) / stride in
  loop (start + (stride * (num_elts - 1))) num_elts []
;;

(* Standard functions *)
let length = List.length
let hd_exn = List.hd
let tl_exn = List.tl

let hd t =
  match t with
  | [] -> None
  | x :: _ -> Some x
;;

let tl t =
  match t with
  | [] -> None
  | _ :: t' -> Some t'
;;

let nth t n =
  if n < 0 then None else
  let rec nth_aux t n =
    match t with
    | [] -> None
    | a :: t -> if n = 0 then Some a else nth_aux t (n-1)
  in nth_aux t n
;;

let nth_exn t n =
  match nth t n with
  | None ->
      invalid_argf "List.nth_exn %d called on list of length %d"
        n (length t) ()
  | Some a -> a
;;

let rev_append = List.rev_append

let rev = function
  | [] | [_] as res -> res
  | x :: y :: rest -> rev_append rest [y; x]

let unordered_append l1 l2 =
  match l1, l2 with
  | [], l | l, [] -> l
  | _             -> List.rev_append l1 l2

let rev_map t ~f = List.rev_map t ~f

exception Length_mismatch of string * int * int

let check_length2 name l1 l2 =
  let n1 = length l1 in
  let n2 = length l2 in
  if n1 <> n2 then
    raise (invalid_argf "length mismatch in %s: %d <> %d " name n1 n2 ())
;;

let check_length3 name l1 l2 l3 =
  let n1 = length l1 in
  let n2 = length l2 in
  let n3 = length l3 in
  if n1 <> n2 || n2 <> n3 then
    raise (invalid_argf "length mismatch in %s: %d <> %d || %d <> %d"
             name n1 n2 n2 n3 ())
;;

let iter2_exn l1 l2 ~f =
  check_length2 "iter2_exn" l1 l2;
  List.iter2 l1 l2 ~f;
;;

let rev_map2_exn l1 l2 ~f  =
  check_length2 "rev_map2_exn" l1 l2;
  List.rev_map2 l1 l2 ~f;
;;

let fold2_exn l1 l2 ~init ~f =
  check_length2 "fold2_exn" l1 l2;
  List.fold_left2 l1 l2 ~init ~f;
;;

let for_all2_exn l1 l2 ~f =
  check_length2 "for_all2_exn" l1 l2;
  List.for_all2 l1 l2 ~f;
;;

let exists2_exn l1 l2 ~f =
  check_length2 "exists2_exn" l1 l2;
  List.exists2 l1 l2 ~f;
;;

let mem ?(equal = (=)) t a = List.exists t ~f:(equal a)

(* This is a copy of the standard library assq function. *)
let rec assq x = function
    [] -> raise Not_found
  | (a,b)::l -> if a == x then b else assq x l

(* This is a copy of the code from the standard library, with an extra eta-expansion to
   avoid creating partial closures (showed up for List.filter in profiling). *)
let rev_filter t ~f =
  let rec find ~f accu = function
    | [] -> accu
    | x :: l -> if f x then find ~f (x :: accu) l else find ~f accu l
  in
  find ~f [] t
;;
<commit_msg>[hack] add dual list utilities to core_list, since these exist in the actual core list module<commit_after>module UnlabeledList = List
module List = StdLabels.List
module String = StdLabels.String

let invalid_argf = Core_printf.invalid_argf

module T = struct
  type 'a t = 'a list
end

include T

module Or_unequal_lengths = struct
  type 'a t =
    | Ok of 'a
    | Unequal_lengths
end

let of_list t = t

let range ?(stride=1) ?(start=`inclusive) ?(stop=`exclusive) start_i stop_i =
  if stride = 0 then
    invalid_arg "Core_list.range: stride must be non-zero";
  (* Generate the range from the last element, so that we do not need to rev it *)
  let rec loop last counter accum =
    if counter <= 0 then accum
    else loop (last - stride) (counter - 1) (last :: accum)
  in
  let stride_sign = if stride > 0 then 1 else -1 in
  let start =
    match start with
    | `inclusive -> start_i
    | `exclusive -> start_i + stride
  in
  let stop =
    match stop with
    | `inclusive -> stop_i + stride_sign
    | `exclusive -> stop_i
  in
  let num_elts = (stop - start + stride - stride_sign) / stride in
  loop (start + (stride * (num_elts - 1))) num_elts []
;;

(* Standard functions *)
let length = List.length
let hd_exn = List.hd
let tl_exn = List.tl

let hd t =
  match t with
  | [] -> None
  | x :: _ -> Some x
;;

let tl t =
  match t with
  | [] -> None
  | _ :: t' -> Some t'
;;

let nth t n =
  if n < 0 then None else
  let rec nth_aux t n =
    match t with
    | [] -> None
    | a :: t -> if n = 0 then Some a else nth_aux t (n-1)
  in nth_aux t n
;;

let nth_exn t n =
  match nth t n with
  | None ->
      invalid_argf "List.nth_exn %d called on list of length %d"
        n (length t) ()
  | Some a -> a
;;

let rev_append = List.rev_append

let rev = function
  | [] | [_] as res -> res
  | x :: y :: rest -> rev_append rest [y; x]

let unordered_append l1 l2 =
  match l1, l2 with
  | [], l | l, [] -> l
  | _             -> List.rev_append l1 l2

let rev_map t ~f = List.rev_map t ~f

exception Length_mismatch of string * int * int

let check_length2_exn name l1 l2 =
  let n1 = length l1 in
  let n2 = length l2 in
  if n1 <> n2 then
    raise (invalid_argf "length mismatch in %s: %d <> %d " name n1 n2 ())
;;

let check_length2 l1 l2 ~f =
  if length l1 <> length l2
  then Or_unequal_lengths.Unequal_lengths
  else Or_unequal_lengths.Ok (f l1 l2)
;;

let check_length3 name l1 l2 l3 =
  let n1 = length l1 in
  let n2 = length l2 in
  let n3 = length l3 in
  if n1 <> n2 || n2 <> n3 then
    raise (invalid_argf "length mismatch in %s: %d <> %d || %d <> %d"
             name n1 n2 n2 n3 ())
;;

let iter2 l1 l2 ~f = check_length2 l1 l2 ~f:(List.iter2 ~f)

let iter2_exn l1 l2 ~f =
  check_length2_exn "iter2_exn" l1 l2;
  List.iter2 l1 l2 ~f;
;;

let rev_map2 l1 l2 ~f = check_length2 l1 l2 ~f:(List.rev_map2 ~f)

let rev_map2_exn l1 l2 ~f  =
  check_length2_exn "rev_map2_exn" l1 l2;
  List.rev_map2 l1 l2 ~f;
;;

let fold2 l1 l2 ~init ~f = check_length2 l1 l2 ~f:(List.fold_left2 ~init ~f)

let fold2_exn l1 l2 ~init ~f =
  check_length2_exn "fold2_exn" l1 l2;
  List.fold_left2 l1 l2 ~init ~f;
;;

let for_all2 l1 l2 ~f = check_length2 l1 l2 ~f:(List.for_all2 ~f)

let for_all2_exn l1 l2 ~f =
  check_length2_exn "for_all2_exn" l1 l2;
  List.for_all2 l1 l2 ~f;
;;

let exists2 l1 l2 ~f = check_length2 l1 l2 ~f:(List.exists2 ~f)

let exists2_exn l1 l2 ~f =
  check_length2_exn "exists2_exn" l1 l2;
  List.exists2 l1 l2 ~f;
;;

let mem ?(equal = (=)) t a = List.exists t ~f:(equal a)

(* This is a copy of the standard library assq function. *)
let rec assq x = function
    [] -> raise Not_found
  | (a,b)::l -> if a == x then b else assq x l

(* This is a copy of the code from the standard library, with an extra eta-expansion to
   avoid creating partial closures (showed up for List.filter in profiling). *)
let rev_filter t ~f =
  let rec find ~f accu = function
    | [] -> accu
    | x :: l -> if f x then find ~f (x :: accu) l else find ~f accu l
  in
  find ~f [] t
;;

{'fraction_non_alphanumeric': 0.11289227007200109, 'fraction_numerical': 0.03178915908164651, 'mean_word_length': 2.662686567164179, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3203603', 'n_tokens_mistral': 2980, 'n_tokens_neox': 2928, 'n_words': 1079}