<commit_before>(* Js_of_ocaml compiler
 * http://www.ocsigen.org/js_of_ocaml/
 * Copyright (C) 2010 <NAME>
 * Laboratoire PPS - CNRS Université Paris Diderot
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, with linking exception;
 * either version 2.1 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*)

module Int = struct type t = int let compare (x : int) y = compare x y end
module IntSet = Set.Make (Int)
module IntMap = Map.Make (Int)

module StringSet = Set.Make (String)
module StringMap = Map.Make (String)

let opt_map f x = match x with None -> None | Some v -> Some (f v)
let opt_iter f x = match x with None -> () | Some v -> f v
let opt_filter p x =
  match x with None -> None | Some v -> if p v then Some v else None

(****)
let quiet = ref false
let warn fmt = Format.ksprintf (fun s -> if not !quiet then Format.eprintf "%s%!" s) fmt

let find_pkg_dir_ref = ref (fun _ -> raise Not_found)
let set_find_pkg_dir f = find_pkg_dir_ref:=f
let find_pkg_dir pkg = !find_pkg_dir_ref pkg

let path_require_findlib path =
  if path <> "" && path.[0] = '+'
  then Some (String.sub  path 1 (String.length path - 1))
  else None

let rec find_in_findlib_paths ?(pkg="stdlib") paths name =
  match paths with
  | [] ->
    raise Not_found
  | path :: rem ->
    try
      let file = match path_require_findlib path with
        | Some path ->
          Filename.concat (Filename.concat (find_pkg_dir pkg) path) name
        | None -> Filename.concat path name in

      if Sys.file_exists file then file else
        find_in_findlib_paths rem name
    with Not_found -> find_in_findlib_paths rem name

let rec find_in_path paths name =
  match paths with
  | [] -> raise Not_found
  | path :: rem ->
    let file = Filename.concat path name in
    if Sys.file_exists file
    then file
    else find_in_path rem name

let absolute_path f =
  if Filename.is_relative f
  then Filename.concat (Sys.getcwd()) f
  else f

let read_file f =
  let ic = open_in f in
  let n = in_channel_length ic in
  let s = Bytes.create n in
  really_input ic s 0 n;
  close_in ic;
  Bytes.unsafe_to_string s

let filter_map f l =
  let l = List.fold_left (fun acc x -> match f x with
    | Some x -> x::acc
    | None -> acc) [] l
  in List.rev l

let sort_uniq compare l =
  let l = List.sort compare l in<commit_msg>Compiler: fix #328<commit_after>(* Js_of_ocaml compiler
 * http://www.ocsigen.org/js_of_ocaml/
 * Copyright (C) 2010 <NAME>
 * Laboratoire PPS - CNRS Université Paris Diderot
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, with linking exception;
 * either version 2.1 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*)

module Int = struct type t = int let compare (x : int) y = compare x y end
module IntSet = Set.Make (Int)
module IntMap = Map.Make (Int)

module StringSet = Set.Make (String)
module StringMap = Map.Make (String)

let opt_map f x = match x with None -> None | Some v -> Some (f v)
let opt_iter f x = match x with None -> () | Some v -> f v
let opt_filter p x =
  match x with None -> None | Some v -> if p v then Some v else None

(****)
let quiet = ref false
let warn fmt = Format.ksprintf (fun s -> if not !quiet then Format.eprintf "%s%!" s) fmt

let find_pkg_dir_ref = ref (fun _ -> raise Not_found)
let set_find_pkg_dir f = find_pkg_dir_ref:=f
let find_pkg_dir pkg = !find_pkg_dir_ref pkg

let path_require_findlib path =
  if path <> "" && path.[0] = '+'
  then Some (String.sub  path 1 (String.length path - 1))
  else None

let rec find_in_findlib_paths ?(pkg="stdlib") paths name =
  match paths with
  | [] ->
    raise Not_found
  | path :: rem ->
    try
      let file = match path_require_findlib path with
        | Some path ->
          Filename.concat (Filename.concat (find_pkg_dir pkg) path) name
        | None -> Filename.concat path name in

      if Sys.file_exists file then file else
        find_in_findlib_paths rem name
    with Not_found -> find_in_findlib_paths rem name

let rec find_in_path paths name =
  match paths with
  | [] -> raise Not_found
  | path :: rem ->
    let file = Filename.concat path name in
    if Sys.file_exists file
    then file
    else find_in_path rem name

let absolute_path f =
  if Filename.is_relative f
  then Filename.concat (Sys.getcwd()) f
  else f

let read_file f =
  let ic = open_in_bin f in
  let n = in_channel_length ic in
  let s = Bytes.create n in
  really_input ic s 0 n;
  close_in ic;
  Bytes.unsafe_to_string s

let filter_map f l =
  let l = List.fold_left (fun acc x -> match f x with
    | Some x -> x::acc
    | None -> acc) [] l
  in List.rev l

let sort_uniq compare l =
  let l = List.sort compare l in
{'fraction_non_alphanumeric': 0.08226659839563065, 'fraction_numerical': 0.008704557091653867, 'mean_word_length': 3.1065171688857744, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '770926', 'n_tokens_mistral': 1909, 'n_tokens_neox': 1834, 'n_words': 809}