<commit_before>(* Misc utils *)

let debug = Trace.debug "misc"

(* ------------- additional Hashtbl utilities --------------- *)
let safe_hash_add tbl x y =
  if (Hashtbl.mem tbl x) then
    raise (Failure "Misc.hash_safe_add: key already bound")
  else
    Hashtbl.add tbl x y

(* --- *)

let enum l =
  Safelist.rev
    (snd (Safelist.fold_left
	    (fun (i,acc) k -> (i+1,(i,k)::acc))
	    (0,[]) l))

(* determines if all the elements of l are unique *)
let uniq l =
  let rec loop = function
      [] -> true
    | [x] -> true
    | (x::y::rest) -> x <> y && (loop (y::rest)) in
  loop (Safelist.sort compare l)

let rec union l1 l2 =
  match l1 with
    [] -> l2
  | h::t -> if Safelist.mem h l2 then union t l2 else h :: (union t l2)

let rec remove n l =
  match l with
    [] -> l
  | h::t -> if h=n then t else h::(remove n t)

let safeheadtail = function
    h::t -> Some h, t
  | [] -> None, []

let safetail l = try
  Safelist.tl l
with (Failure "tl") -> []

let map_option f = function
    None -> None
  | Some x -> Some (f x)

let rec fold_left2 f c xs ys = match xs,ys with
  [],[] -> c
| (x::xs),[] -> fold_left2 f (f c (Some x) None) xs []
| [],(y::ys) -> fold_left2 f (f c None (Some y)) [] ys
| (x::xs),(y::ys) -> fold_left2 f (f c (Some x) (Some y)) xs ys

let rec fold_left3 f c xs ys zs = match xs,ys,zs with
  [],[],[] -> c
| (x::xs),[],[] -> fold_left3 f (f c (Some x) None None) xs [] []
| [],(y::ys), [] -> fold_left3 f (f c None (Some y) None) [] ys []
| [],[], (z::zs) -> fold_left3 f (f c None None (Some z)) [] [] zs
| (x::xs),(y::ys), [] -> fold_left3 f (f c (Some x) (Some y) None) xs ys []
| (x::xs),[], (z::zs) -> fold_left3 f (f c (Some x) None (Some z)) xs [] zs
| [], (y::ys), (z::zs) -> fold_left3 f (f c None (Some y) (Some z)) [] ys zs
| (x::xs), (y::ys), (z::zs) -> fold_left3 f (f c (Some x) (Some y) (Some z)) xs ys zs

let fold_left2_with_pad f c xs ys padx pady =
  let rec loop acc = function
      [],[] -> acc
    | (x::xs),[] -> loop (f acc x pady) (xs,[])
    | [],(y::ys) -> loop (f acc padx y) ([],ys)
    | (x::xs),(y::ys) -> loop (f acc x y) (xs,ys) in
  loop c (xs,ys)

let map2opt f xs ys =
  let rec loop acc = function
      [],[] -> Safelist.rev acc
    | x::xs,[] -> loop ((f (Some x) None)::acc) (xs,[])
    | [],y::ys -> loop ((f None (Some y))::acc) ([],ys)
    | x::xs,y::ys -> loop ((f (Some x) (Some y))::acc) (xs,ys) in
  loop [] (xs,ys)
  

let map2_with_pad f xs ys padx pady =
  Safelist.rev
    (fold_left2_with_pad
       (fun acc x y -> (f x y)::acc)
       [] xs ys padx pady)

let zip_with_pad pad1 pad2 l1 l2 =
  let rec loop acc = function
      [],[] -> Safelist.rev acc
    | x::xs,[] -> loop ((x,pad2)::acc) (xs,[])
    | [],y::ys -> loop ((pad1,y)::acc) ([],ys)
    | x::xs,y::ys -> loop ((x,y)::acc) (xs,ys) in
  loop [] (l1,l2)

let rec iter_with_sep f fsep = function
    [] -> ()
  | [x] -> f x
  | (x::xs) -> f x; fsep (); iter_with_sep f fsep xs

let rev_and_flatten ll =
  let rec help acc = function
      [] -> acc
    | (l::ls) -> help (Safelist.append l acc) ls in
  help [] ll
  
let safeassoc alist =
  fun k ->
    (try
      Safelist.assoc k alist
    with Not_found -> k)

(* partition n l divides the list l into two segments, where the first*)
(* segment has length n. if there are not n elements in l, all the *)
(* elements go into the first segment and the second segment is empty *)
let partition n l =
  let rec help n l acc =
    match n,l with
      0,_ -> (Safelist.rev acc,l)
    | n,(x::xs) -> help (n-1) xs (x::acc)
    | _,_ -> assert false in
  if ((Safelist.length l) <= n) then (l,[]) else
  help n l []

(* composel : ('a -> 'a) list -> ('a -> 'a) *)
let composel (l : ('a -> 'a) list) =
  Safelist.fold_left (fun f acc -> (fun s -> f (acc s))) (fun x -> x) l

(* --- *)

exception Bad of string

let bad s = raise (Bad s)

exception Unimplemented of string

(* --- *)

(* Based on String.escape 
 *)
let escape (escapeChar: char -> string) s =
  debug (fun () -> Format.printf "escape: %s\n" s);
  let n = ref 0 in
    for i = 0 to String. length s - 1 do
      let l = String.length (escapeChar (String.get s i)) in
      assert (l>0);
      n := !n + l;
    done;
  let result = 
    if !n = String.length s then s else 
    begin
      let s' = String.create !n in   
      n := 0;
      for i = 0 to String.length s - 1 do 
	let c = String.get s i in
	let cEscaped = escapeChar c in
	if String.length cEscaped = 1 then begin
	  String.set s' !n c; incr n
	end else
	  for i = 0 to String.length cEscaped - 1 do begin
	    String.set s' !n (String.get cEscaped i);
	    incr n;
	  end done
      done;
      s'
    end
  in
  debug(fun () -> Format.printf "escape returns %s\n" result);
  result

(* \ -> \\; x -> \x (for all x in escapedchars) *)
let generic_escape_char escapedchars c = 
  if c = '\\' then 
    "\\\\"
  else if String.contains escapedchars c then
    let str = String.create 2 in
    String.set str 0 '\\';
    String.set str 1 c;
    str
  else
    "-"

(* \x -> x *)
let generic_unescape s =
  if not (String.contains s '\\') then
    s
  else
    let rec loop i n =
      if i >= String.length s then
	n
      else if s.[i] = '\\' then
	loop (i + 2) (n + 1)
      else
	loop (i + 1) (n + 1)
    in
    let s' = String.create (loop 0 0) in
    let rec loop i i' = 
      if i >= String.length s then
	()
      else if s.[i] = '\\' then begin
	s'.[i'] <- s.[i+1]; (* assumes that '\' always followed by a char *)
	loop (i + 2) (i' + 1)
      end else begin
	s'.[i'] <- s.[i];
	loop (i + 1) (i' + 1)
      end
    in
    loop 0 0;
    s'
      
(* find c, skipping all the escaped characters, e.g., "\;" *)
let rec index_rec_nonescape s i c = 
  if i >= String.length s then 
    raise Not_found 
  else if String.unsafe_get s i = c then 
    i
  else if String.unsafe_get s i = '\\' then
    index_rec_nonescape s (i+2) c
  else
    index_rec_nonescape s (i+1) c
      
let split_nonescape c text = 
  let rec split start =
    try 
      let pos = index_rec_nonescape text start c in
      String.sub text start (pos-start) :: split (pos+1)
    with Not_found ->
      [String.sub text start (String.length text - start)]
  in
  split 0

let is_blank s = 
  let l = String.length s in
  let rec loop i =
    if i = l then true
    else String.contains "\n \t" s.[i] && loop (i + 1)
  in
  loop 0

let bounded_index_from buf pos len char =
  let endpos = pos + len in
  let rec loop i =
    if i >= endpos then raise Not_found
    else if buf.[i] = char then i
    else loop (i+1)
  in
  loop pos

let findsubstring s1 s2 =
  let l1 = String.length s1 in
  let l2 = String.length s2 in
  let rec loop i =
    if i+l1 > l2 then None
    else if s1 = String.sub s2 i l1 then Some(i)
    else loop (i+1)
  in loop 0

let trimLeadingSpaces s = 
  let l = String.length s in
  let rec loop i = 
    if i < l && s.[i] = ' ' then loop (i + 1)
    else String.sub s i (l - i)
  in
  loop 0

let rec replace_substring s fromstring tostring =
  match findsubstring fromstring s with
    None -> s
  | Some(i) ->
      let before = String.sub s 0 i in
      let afterpos = i + (String.length fromstring) in
      let after = String.sub s afterpos ((String.length s) - afterpos) in
      before ^ tostring ^ (replace_substring after fromstring tostring)

let rec replace_substrings s l =
  match l with
    [] -> s
  | (sub,sub')::rest -> replace_substrings (replace_substring s sub sub') rest

    
let whack_chars s cl =
  (*   let s = if String.length s > 30 then (String.sub s 0 30 ^ "...") else s in  *)
  let s = String.escaped s in
  let s = 
    if Safelist.exists (fun c -> String.contains s c) cl then "\""^s^"\""
    else s in
    if s="" then "\"\""
    else s
      
let whack s = whack_chars s [' '; '"']

let unescaped s =
  let buf = Buffer.create (String.length s) in
  let rec loop i =
    if (i >= (String.length s)) then () else
    let c = s.[i] in
    if (c = '\\') then
      let c2 = s.[i+1] in
      match c2 with
	'\\' | '"' | '\'' -> (Buffer.add_char buf c2; loop (i+2))
      | 'n' -> (Buffer.add_char buf '\n'; loop (i+2))
      | 'r' -> (Buffer.add_char buf '\r'; loop (i+2))
      | 't' -> (Buffer.add_char buf '\t'; loop (i+2))
      | 'b' -> (Buffer.add_char buf '\b'; loop (i+2))
      | _ ->
	  let c3 = s.[i+2] in
	  let c4 = s.[i+3] in
	  let i2 = int_of_char(c2) - int_of_char('0') in
	  let i3 = int_of_char(c3) - int_of_char('0') in
	  let i4 = int_of_char(c4) - int_of_char('0') in
	  if (i2 < 0 || i2 > 9 || i3 < 0 || i3 > 9 || i4 < 0 || i4 > 9) then<commit_msg>tiny mod in the Misc.whack, to escape only ' and "<commit_after>(* Misc utils *)

let debug = Trace.debug "misc"

(* ------------- additional Hashtbl utilities --------------- *)
let safe_hash_add tbl x y =
  if (Hashtbl.mem tbl x) then
    raise (Failure "Misc.hash_safe_add: key already bound")
  else
    Hashtbl.add tbl x y

(* --- *)

let enum l =
  Safelist.rev
    (snd (Safelist.fold_left
	    (fun (i,acc) k -> (i+1,(i,k)::acc))
	    (0,[]) l))

(* determines if all the elements of l are unique *)
let uniq l =
  let rec loop = function
      [] -> true
    | [x] -> true
    | (x::y::rest) -> x <> y && (loop (y::rest)) in
  loop (Safelist.sort compare l)

let rec union l1 l2 =
  match l1 with
    [] -> l2
  | h::t -> if Safelist.mem h l2 then union t l2 else h :: (union t l2)

let rec remove n l =
  match l with
    [] -> l
  | h::t -> if h=n then t else h::(remove n t)

let safeheadtail = function
    h::t -> Some h, t
  | [] -> None, []

let safetail l = try
  Safelist.tl l
with (Failure "tl") -> []

let map_option f = function
    None -> None
  | Some x -> Some (f x)

let rec fold_left2 f c xs ys = match xs,ys with
  [],[] -> c
| (x::xs),[] -> fold_left2 f (f c (Some x) None) xs []
| [],(y::ys) -> fold_left2 f (f c None (Some y)) [] ys
| (x::xs),(y::ys) -> fold_left2 f (f c (Some x) (Some y)) xs ys

let rec fold_left3 f c xs ys zs = match xs,ys,zs with
  [],[],[] -> c
| (x::xs),[],[] -> fold_left3 f (f c (Some x) None None) xs [] []
| [],(y::ys), [] -> fold_left3 f (f c None (Some y) None) [] ys []
| [],[], (z::zs) -> fold_left3 f (f c None None (Some z)) [] [] zs
| (x::xs),(y::ys), [] -> fold_left3 f (f c (Some x) (Some y) None) xs ys []
| (x::xs),[], (z::zs) -> fold_left3 f (f c (Some x) None (Some z)) xs [] zs
| [], (y::ys), (z::zs) -> fold_left3 f (f c None (Some y) (Some z)) [] ys zs
| (x::xs), (y::ys), (z::zs) -> fold_left3 f (f c (Some x) (Some y) (Some z)) xs ys zs

let fold_left2_with_pad f c xs ys padx pady =
  let rec loop acc = function
      [],[] -> acc
    | (x::xs),[] -> loop (f acc x pady) (xs,[])
    | [],(y::ys) -> loop (f acc padx y) ([],ys)
    | (x::xs),(y::ys) -> loop (f acc x y) (xs,ys) in
  loop c (xs,ys)

let map2opt f xs ys =
  let rec loop acc = function
      [],[] -> Safelist.rev acc
    | x::xs,[] -> loop ((f (Some x) None)::acc) (xs,[])
    | [],y::ys -> loop ((f None (Some y))::acc) ([],ys)
    | x::xs,y::ys -> loop ((f (Some x) (Some y))::acc) (xs,ys) in
  loop [] (xs,ys)
  

let map2_with_pad f xs ys padx pady =
  Safelist.rev
    (fold_left2_with_pad
       (fun acc x y -> (f x y)::acc)
       [] xs ys padx pady)

let zip_with_pad pad1 pad2 l1 l2 =
  let rec loop acc = function
      [],[] -> Safelist.rev acc
    | x::xs,[] -> loop ((x,pad2)::acc) (xs,[])
    | [],y::ys -> loop ((pad1,y)::acc) ([],ys)
    | x::xs,y::ys -> loop ((x,y)::acc) (xs,ys) in
  loop [] (l1,l2)

let rec iter_with_sep f fsep = function
    [] -> ()
  | [x] -> f x
  | (x::xs) -> f x; fsep (); iter_with_sep f fsep xs

let rev_and_flatten ll =
  let rec help acc = function
      [] -> acc
    | (l::ls) -> help (Safelist.append l acc) ls in
  help [] ll
  
let safeassoc alist =
  fun k ->
    (try
      Safelist.assoc k alist
    with Not_found -> k)

(* partition n l divides the list l into two segments, where the first*)
(* segment has length n. if there are not n elements in l, all the *)
(* elements go into the first segment and the second segment is empty *)
let partition n l =
  let rec help n l acc =
    match n,l with
      0,_ -> (Safelist.rev acc,l)
    | n,(x::xs) -> help (n-1) xs (x::acc)
    | _,_ -> assert false in
  if ((Safelist.length l) <= n) then (l,[]) else
  help n l []

(* composel : ('a -> 'a) list -> ('a -> 'a) *)
let composel (l : ('a -> 'a) list) =
  Safelist.fold_left (fun f acc -> (fun s -> f (acc s))) (fun x -> x) l

(* --- *)

exception Bad of string

let bad s = raise (Bad s)

exception Unimplemented of string

(* --- *)

(* Based on String.escape 
 *)
let escape (escapeChar: char -> string) s =
  debug (fun () -> Format.printf "escape: %s\n" s);
  let n = ref 0 in
    for i = 0 to String. length s - 1 do
      let l = String.length (escapeChar (String.get s i)) in
      assert (l>0);
      n := !n + l;
    done;
  let result = 
    if !n = String.length s then s else 
    begin
      let s' = String.create !n in   
      n := 0;
      for i = 0 to String.length s - 1 do 
	let c = String.get s i in
	let cEscaped = escapeChar c in
	if String.length cEscaped = 1 then begin
	  String.set s' !n c; incr n
	end else
	  for i = 0 to String.length cEscaped - 1 do begin
	    String.set s' !n (String.get cEscaped i);
	    incr n;
	  end done
      done;
      s'
    end
  in
  debug(fun () -> Format.printf "escape returns %s\n" result);
  result

(* \ -> \\; x -> \x (for all x in escapedchars) *)
let generic_escape_char escapedchars c = 
  if c = '\\' then 
    "\\\\"
  else if String.contains escapedchars c then
    let str = String.create 2 in
    String.set str 0 '\\';
    String.set str 1 c;
    str
  else
    "-"

(* \x -> x *)
let generic_unescape s =
  if not (String.contains s '\\') then
    s
  else
    let rec loop i n =
      if i >= String.length s then
	n
      else if s.[i] = '\\' then
	loop (i + 2) (n + 1)
      else
	loop (i + 1) (n + 1)
    in
    let s' = String.create (loop 0 0) in
    let rec loop i i' = 
      if i >= String.length s then
	()
      else if s.[i] = '\\' then begin
	s'.[i'] <- s.[i+1]; (* assumes that '\' always followed by a char *)
	loop (i + 2) (i' + 1)
      end else begin
	s'.[i'] <- s.[i];
	loop (i + 1) (i' + 1)
      end
    in
    loop 0 0;
    s'
      
(* find c, skipping all the escaped characters, e.g., "\;" *)
let rec index_rec_nonescape s i c = 
  if i >= String.length s then 
    raise Not_found 
  else if String.unsafe_get s i = c then 
    i
  else if String.unsafe_get s i = '\\' then
    index_rec_nonescape s (i+2) c
  else
    index_rec_nonescape s (i+1) c
      
let split_nonescape c text = 
  let rec split start =
    try 
      let pos = index_rec_nonescape text start c in
      String.sub text start (pos-start) :: split (pos+1)
    with Not_found ->
      [String.sub text start (String.length text - start)]
  in
  split 0

let is_blank s = 
  let l = String.length s in
  let rec loop i =
    if i = l then true
    else String.contains "\n \t" s.[i] && loop (i + 1)
  in
  loop 0

let bounded_index_from buf pos len char =
  let endpos = pos + len in
  let rec loop i =
    if i >= endpos then raise Not_found
    else if buf.[i] = char then i
    else loop (i+1)
  in
  loop pos

let findsubstring s1 s2 =
  let l1 = String.length s1 in
  let l2 = String.length s2 in
  let rec loop i =
    if i+l1 > l2 then None
    else if s1 = String.sub s2 i l1 then Some(i)
    else loop (i+1)
  in loop 0

let trimLeadingSpaces s = 
  let l = String.length s in
  let rec loop i = 
    if i < l && s.[i] = ' ' then loop (i + 1)
    else String.sub s i (l - i)
  in
  loop 0

let rec replace_substring s fromstring tostring =
  match findsubstring fromstring s with
    None -> s
  | Some(i) ->
      let before = String.sub s 0 i in
      let afterpos = i + (String.length fromstring) in
      let after = String.sub s afterpos ((String.length s) - afterpos) in
      before ^ tostring ^ (replace_substring after fromstring tostring)

let rec replace_substrings s l =
  match l with
    [] -> s
  | (sub,sub')::rest -> replace_substrings (replace_substring s sub sub') rest

    
let whack_chars s cl =
  (*   let s = if String.length s > 30 then (String.sub s 0 30 ^ "...") else s in  *)
  let s = 
    (* escape only the strictly necessary *)
    Safelist.fold_right 
      ( fun (p,r) s -> Str.global_replace (Str.regexp p ) r s )   
      [("\"","\\\"");("\'","\\\'")]
      s
  in
  let s = 
    if Safelist.exists (fun c -> String.contains s c) cl then "\""^s^"\""
    else s in
    if s="" then "\"\""
    else s
      
let whack s = whack_chars s [' '; '"']

let unescaped s =
  let buf = Buffer.create (String.length s) in
  let rec loop i =
    if (i >= (String.length s)) then () else
    let c = s.[i] in
    if (c = '\\') then
      let c2 = s.[i+1] in
      match c2 with
	'\\' | '"' | '\'' -> (Buffer.add_char buf c2; loop (i+2))
      | 'n' -> (Buffer.add_char buf '\n'; loop (i+2))
      | 'r' -> (Buffer.add_char buf '\r'; loop (i+2))
      | 't' -> (Buffer.add_char buf '\t'; loop (i+2))
      | 'b' -> (Buffer.add_char buf '\b'; loop (i+2))
      | _ ->
	  let c3 = s.[i+2] in
	  let c4 = s.[i+3] in
	  let i2 = int_of_char(c2) - int_of_char('0') in
	  let i3 = int_of_char(c3) - int_of_char('0') in
	  let i4 = int_of_char(c4) - int_of_char('0') in
	  if (i2 < 0 || i2 > 9 || i3 < 0 || i3 > 9 || i4 < 0 || i4 > 9) then
{'fraction_non_alphanumeric': 0.1691559569294894, 'fraction_numerical': 0.014241055922195206, 'mean_word_length': 2.226559581621218, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 174, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2581913', 'n_tokens_mistral': 6922, 'n_tokens_neox': 6708, 'n_words': 2457}