<commit_before>(*
 * Copyright (c) 2009 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *)

(* Fork of http://xenbits.xen.org/xapi/xen-api-libs.hg?file/7a17b2ab5cfc/rpc-light/rpc.ml *)

open Printf

type t =
	| Int of int64
	| Bool of bool
	| Float of float
	| String of string
	| Enum of t list (* all the t of the list should be of same type *)
	| Tuple of t list
	| Dict of (string * t) list
	| Sum of string * t list
	| Null
	| Value of t
	| Var of (string * int64)
	| Rec of (string * int64) * t
	| Arrow of string
	| Ext of (string * int64) * t
 with weakid

(* If there are still some Var v, then the type is recursive for the type v *)
let free_vars t =
	let rec aux accu = function
	| Rec (n,t)  -> aux (List.filter (fun m -> n <> m) accu) t
	| Var n when List.mem n accu -> accu
	| Var n      -> n :: accu
	| Value t     -> aux accu t
	| Enum tl
	| Sum (_,tl)
	| Tuple tl   -> List.fold_left aux accu tl
	| Dict tl    -> List.fold_left (fun accu (_,t) -> aux accu t) accu tl
	| Int _ | Bool _ | Float _ | String _ | Null | Arrow _
                     -> accu
	| Ext (_,t)  -> aux accu t in
	aux [] t

let map_strings sep fn l = String.concat sep (List.map fn l)

let rec to_string t = match t with                                                                    
	| Null       -> "N"
	| Value t    -> sprintf "?%s" (to_string t)<commit_msg>[value] add an 'equal' function which doesn't care about the exact indexes of recursive/external types.<commit_after>(*
 * Copyright (c) 2009 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *)

(* Fork of http://xenbits.xen.org/xapi/xen-api-libs.hg?file/7a17b2ab5cfc/rpc-light/rpc.ml *)

open Printf

type t =
	| Int of int64
	| Bool of bool
	| Float of float
	| String of string
	| Enum of t list (* all the t of the list should be of same type *)
	| Tuple of t list
	| Dict of (string * t) list
	| Sum of string * t list
	| Null
	| Value of t
	| Var of (string * int64)
	| Rec of (string * int64) * t
	| Arrow of string
	| Ext of (string * int64) * t
 with weakid

(* If there are still some Var v, then the type is recursive for the type v *)
let free_vars t =
	let rec aux accu = function
	| Rec (n,t)  -> aux (List.filter (fun m -> n <> m) accu) t
	| Var n when List.mem n accu -> accu
	| Var n      -> n :: accu
	| Value t     -> aux accu t
	| Enum tl
	| Sum (_,tl)
	| Tuple tl   -> List.fold_left aux accu tl
	| Dict tl    -> List.fold_left (fun accu (_,t) -> aux accu t) accu tl
	| Int _ | Bool _ | Float _ | String _ | Null | Arrow _
                     -> accu
	| Ext (_,t)  -> aux accu t in
	aux [] t

let map_strings sep fn l = String.concat sep (List.map fn l)

let equal x y =
	let rec aux ids x y = match x, y with
	| Int i       , Int j       -> i=j
	| Bool i      , Bool j      -> i=j
	| Float i     , Float j     -> i=j
	| String i    , String j    -> i=j
	| Enum i      , Enum j
	| Tuple i     , Tuple j     -> List.for_all (fun x -> x) (List.map2 (aux ids) i j)
	| Dict i      , Dict j      ->
		let fn a b = List.for_all (fun (n,i) -> List.exists (fun (m,j) -> n=m && (aux ids) i j) a) b in
		fn i j && fn j i
	| Sum (n,i)   , Sum (m,j)   -> n=m && List.for_all (fun x -> x) (List.map2 (aux ids) i j)
	| Null        , Null        -> true
	| Value i     , Value j     -> aux ids i j
	| Var (n,i)   , Var (m,j)   -> n=m && i = List.assoc j ids
	| Rec((n,i),v), Rec((m,j),w)-> n=m && aux ( (j,i) :: ids ) v w
	| Arrow _     , Arrow _     -> true
	| Ext((n,i),v), Ext((m,j),w)-> n=m && aux ids v w
	| _                         -> false in
	aux [] x y

let rec to_string t = match t with                                                                    
	| Null       -> "N"
	| Value t    -> sprintf "?%s" (to_string t)
{'fraction_non_alphanumeric': 0.10805500982318271, 'fraction_numerical': 0.007072691552062868, 'mean_word_length': 2.3079922027290447, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3775150', 'n_tokens_mistral': 1792, 'n_tokens_neox': 1701, 'n_words': 747}