<commit_before>(*
 * BatEnum - Enumeration over abstract collection of elements.
 * Copyright (C) 2003 <NAME>
 *               2009 <NAME>, LIFO, Universite d'Orleans
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version,
 * with the special exception on linking described in file LICENSE.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *)

(** {6 Representation} *)

type 'a t = {
  mutable count : unit -> int; (** Return the number of remaining elements in the enumeration. *)
  mutable next  : unit -> 'a;  (** Return the next element of the enumeration or raise [No_more_elements].*)
  mutable clone : unit -> 'a t;(** Return a copy of the enumeration. *)
  mutable fast  : bool;        (** [true] if [count] can be done without reading all elements, [false] otherwise.*)
}

type 'a enumerable = 'a t
type 'a mappable = 'a t

external enum : 'a t -> 'a t = "%identity"
external of_enum : 'a t -> 'a t = "%identity"

(* raised by 'next' functions, should NOT go outside the API *)
exception No_more_elements

let make ~next ~count ~clone =
  {
    count = count;
    next  = next;
    clone = clone;
    fast  = true;
  }

(** {6 Internal utilities}*)
let _dummy () = assert false (*BISECT-VISIT*)

(* raised by 'count' functions, may go outside the API *)
exception Infinite_enum

let return_no_more_elements () = raise No_more_elements
let return_no_more_count    () = 0
let return_infinite_count   () = raise Infinite_enum

(* Inlined from ExtList to avoid circular dependencies. *)
type 'a _mut_list = {
	hd : 'a;
	mutable tl : 'a _mut_list;
}

let rec empty () =
  {
    count = return_no_more_count;
    next  = return_no_more_elements;
    clone = empty;
    fast  = true;
  }

let close e =
  e.next <- return_no_more_elements;
  e.count<- return_no_more_count;
  e.clone<- empty

let force t =(** Transform [t] into a list *)
  let rec clone enum count =
    let enum = ref !enum
    and	count = ref !count in
      {
	count = (fun () -> !count);
	next  = (fun () ->
		   match !enum with
		     | []     -> raise No_more_elements
		     | h :: t -> decr count; enum := t; h);
	clone = (fun () ->
		   let enum = ref !enum
		   and count = ref !count in
		     clone enum count);
	fast  = true;
      }
  in
  let count = ref 0 in
  let _empty = Obj.magic [] in
  let rec loop dst =
    let x = { hd = t.next(); tl = _empty } in
      incr count;
      dst.tl <- x;
      loop x
  in
  let enum = ref _empty  in
    (try
       enum := { hd = t.next(); tl = _empty };
       incr count;
       loop !enum;
     with No_more_elements -> ());
    let tc = clone (Obj.magic enum) count in
      t.clone <- tc.clone;
      t.next <- tc.next;
      t.count <- tc.count;
      t.fast <- true

(* Inlined from {!LazyList}.

   This lazy list permits cloning enumerations constructed with {!from}
   without having to actually force them.*)
module MicroLazyList = struct
  type 'a ll_t      = ('a node_t) Lazy.t
  and  'a node_t =
    | Nil
    | Cons of 'a * 'a ll_t

  let nil = lazy Nil

  let enum l =
    let rec aux (l:'a ll_t) : 'a t=
      let reference = ref l in
      let e = make
	~next:(fun () -> match Lazy.force !reference with
		 | Cons(x,t) -> reference := t; x
		 | Nil       -> raise No_more_elements )
        ~count:_dummy
        ~clone:(fun () -> aux !reference)
      in e.count <- (fun () -> force e; e.count());
         e.fast  <- false;
         e
    in aux l

  let from f =
    let rec aux () =
      lazy (
	let item = try  Some (f ())
		   with No_more_elements -> None
	in match item with
	  | Some x -> Cons (x, aux () )
	  | _      -> Nil
      )
    in
      aux ()
end

let from f =
  let e = {
    next = f;
    count = _dummy;
    clone = _dummy;
    fast = false;
  } in
    e.count <- (fun () -> force e; e.count());
    e.clone <- (fun () ->
		  let e' =  MicroLazyList.enum(MicroLazyList.from f) in
		    e.next <- e'.next;
		    e.clone<- e'.clone;
		    e.count<- (fun () -> force e; e.count());
                    (* we can't use [e'.count] because that would force [e'],
                       which doesn't update [e].  That would for example,
                       cause e.fast to not be updated to true.  A simple test
                       to see the problem with [e'.count] is to do the
                       following: (1) create a enum using this [from]
                       function, (2) clone that enum, (3) grab the count of
                       the original enum and then iterate over it.  A
                       discrepancy between the count and the elements will
                       result. *)<commit_msg>Close enums as soon as No_more_elements is raised.<commit_after>(*
 * BatEnum - Enumeration over abstract collection of elements.
 * Copyright (C) 2003 <NAME>
 *               2009 <NAME>, LIFO, Universite d'Orleans
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version,
 * with the special exception on linking described in file LICENSE.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *)

(** {6 Representation} *)

type 'a t = {
  mutable count : unit -> int; (** Return the number of remaining elements in the enumeration. *)
  mutable next  : unit -> 'a;  (** Return the next element of the enumeration or raise [No_more_elements].*)
  mutable clone : unit -> 'a t;(** Return a copy of the enumeration. *)
  mutable fast  : bool;        (** [true] if [count] can be done without reading all elements, [false] otherwise.*)
}

type 'a enumerable = 'a t
type 'a mappable = 'a t

external enum : 'a t -> 'a t = "%identity"
external of_enum : 'a t -> 'a t = "%identity"

(* raised by 'next' functions, should NOT go outside the API *)
exception No_more_elements

let make ~next ~count ~clone =
  {
    count = count;
    next  = next;
    clone = clone;
    fast  = true;
  }

(** {6 Internal utilities}*)
let _dummy () = assert false (*BISECT-VISIT*)

(* raised by 'count' functions, may go outside the API *)
exception Infinite_enum

let return_no_more_elements () = raise No_more_elements
let return_no_more_count    () = 0
let return_infinite_count   () = raise Infinite_enum

(* Inlined from ExtList to avoid circular dependencies. *)
type 'a _mut_list = {
	hd : 'a;
	mutable tl : 'a _mut_list;
}

let rec empty () =
  {
    count = return_no_more_count;
    next  = return_no_more_elements;
    clone = empty;
    fast  = true;
  }

let close e =
  e.next <- return_no_more_elements;
  e.count<- return_no_more_count;
  e.clone<- empty

let force t =(** Transform [t] into a list *)
  let rec clone enum count =
    let enum = ref !enum
    and	count = ref !count in
      {
	count = (fun () -> !count);
	next  = (fun () ->
		   match !enum with
		     | []     -> raise No_more_elements
		     | h :: t -> decr count; enum := t; h);
	clone = (fun () ->
		   let enum = ref !enum
		   and count = ref !count in
		     clone enum count);
	fast  = true;
      }
  in
  let count = ref 0 in
  let _empty = Obj.magic [] in
  let rec loop dst =
    let x = { hd = t.next(); tl = _empty } in
      incr count;
      dst.tl <- x;
      loop x
  in
  let enum = ref _empty  in
    (try
       enum := { hd = t.next(); tl = _empty };
       incr count;
       loop !enum;
     with No_more_elements -> ());
    let tc = clone (Obj.magic enum) count in
      t.clone <- tc.clone;
      t.next <- tc.next;
      t.count <- tc.count;
      t.fast <- true

(* Inlined from {!LazyList}.

   This lazy list permits cloning enumerations constructed with {!from}
   without having to actually force them.*)
module MicroLazyList = struct
  type 'a ll_t      = ('a node_t) Lazy.t
  and  'a node_t =
    | Nil
    | Cons of 'a * 'a ll_t

  let nil = lazy Nil

  let enum l =
    let rec aux (l:'a ll_t) : 'a t=
      let reference = ref l in
      let e = make
	~next:(fun () -> match Lazy.force !reference with
		 | Cons(x,t) -> reference := t; x
		 | Nil       -> raise No_more_elements )
        ~count:_dummy
        ~clone:(fun () -> aux !reference)
      in e.count <- (fun () -> force e; e.count());
         e.fast  <- false;
         e
    in aux l

  let from f =
    let rec aux () =
      lazy (
	let item = try  Some (f ())
		   with No_more_elements -> None
	in match item with
	  | Some x -> Cons (x, aux () )
	  | _      -> Nil
      )
    in
      aux ()
end

let from f =
  let e = {
    next = _dummy;
    count = _dummy;
    clone = _dummy;
    fast = false;
  } in
    e.next  <- (fun () -> try f () with No_more_elements -> close e ; raise No_more_elements);
    e.count <- (fun () -> force e; e.count());
    e.clone <- (fun () ->
		  let e' =  MicroLazyList.enum(MicroLazyList.from f) in
		    e.next <- e'.next;
		    e.clone<- e'.clone;
		    e.count<- (fun () -> force e; e.count());
                    (* we can't use [e'.count] because that would force [e'],
                       which doesn't update [e].  That would for example,
                       cause e.fast to not be updated to true.  A simple test
                       to see the problem with [e'.count] is to do the
                       following: (1) create a enum using this [from]
                       function, (2) clone that enum, (3) grab the count of
                       the original enum and then iterate over it.  A
                       discrepancy between the count and the elements will
                       result. *)
{'fraction_non_alphanumeric': 0.11038654191475286, 'fraction_numerical': 0.005859559587940649, 'mean_word_length': 2.2008469449485784, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1147573', 'n_tokens_mistral': 3450, 'n_tokens_neox': 3256, 'n_words': 1325}