<commit_before>(*
 * Copyright (c) 2015, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the "hack" directory of this source tree.
 *
 *)

(*****************************************************************************)
(* Prelude *)
(*****************************************************************************)

open Hh_core

external native_hh_readdir : string -> (string * int) list = "hh_readdir"

type dt_kind =
  | DT_REG
  | DT_DIR

(* Sys.readdir only returns `string list`, but we need to know if we have files
 * or directories, so if we use Sys.readdir we need to do an lstat on every
 * file/subdirectory. The C readdir function gives us both the name and kind,
 * so this version does 1 syscall per directory, instead of 1 syscall per file
 * and 2 per directory.
 *)
let hh_readdir path : (string * dt_kind) list =
  List.filter_map (native_hh_readdir path) (fun (name, kind) ->
      match (name, kind) with
      | (".", _) -> None
      | ("..", _) -> None
      (* values from `man dirent` *)
      | (_, 4) -> Some (name, DT_DIR)
      | (_, 8) -> Some (name, DT_REG)
      | _ -> None)

let lstat_kind file =
  Unix.(
    try Some (lstat file).st_kind
    with Unix_error (ENOENT, _, _) ->
      prerr_endline ("File not found: " ^ file);
      None)

let fold_files
    (type t)
    ?max_depth
    ?(filter = (fun _ -> true))
    ?(file_only = false)
    (paths : Path.t list)
    (action : string -> t -> t)
    (init : t) =
  let rec fold depth acc dir =
    let acc =
      if (not file_only) && filter dir then
        action dir acc
      else
        acc
    in<commit_msg>[hack] Use find.ml in the sqlite index buidler instead of shelling out<commit_after>(*
 * Copyright (c) 2015, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the "hack" directory of this source tree.
 *
 *)

(*****************************************************************************)
(* Prelude *)
(*****************************************************************************)

open Hh_core

external native_hh_readdir : string -> (string * int) list = "hh_readdir"

type dt_kind =
  | DT_REG
  | DT_DIR

(* Sys.readdir only returns `string list`, but we need to know if we have files
 * or directories, so if we use Sys.readdir we need to do an lstat on every
 * file/subdirectory. The C readdir function gives us both the name and kind,
 * so this version does 1 syscall per directory, instead of 1 syscall per file
 * and 2 per directory.
 *)
let hh_readdir path : (string * dt_kind) list =
  List.filter_map (native_hh_readdir path) (fun (name, kind) ->
      match (name, kind) with
      | (".", _) -> None
      | ("..", _) -> None
      | (".git", _) -> None
      | (".hg", _) -> None
      (* values from `man dirent` *)
      | (_, 4) -> Some (name, DT_DIR)
      | (_, 8) -> Some (name, DT_REG)
      | _ -> None)

let lstat_kind file =
  Unix.(
    try Some (lstat file).st_kind
    with Unix_error (ENOENT, _, _) ->
      prerr_endline ("File not found: " ^ file);
      None)

let fold_files
    (type t)
    ?max_depth
    ?(filter = (fun _ -> true))
    ?(file_only = false)
    (paths : Path.t list)
    (action : string -> t -> t)
    (init : t) =
  let rec fold depth acc dir =
    let acc =
      if (not file_only) && filter dir then
        action dir acc
      else
        acc
    in
{'fraction_non_alphanumeric': 0.1989061600460564, 'fraction_numerical': 0.0051813471502590676, 'mean_word_length': 2.806133625410734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '869876', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1058, 'n_words': 396}