<commit_before>(*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 *)

module Char = struct
  include Char

  let is_lowercase = function 'a' .. 'z' -> true | _ -> false
  let is_uppercase = function 'A' .. 'Z' -> true | _ -> false
end

module String = struct
  include StringLabels

  let is_empty str = length str = 0

  let lsplit2 str ~on =
    match index_opt str on with
    | Some pos ->
        Some
          ( sub str ~pos:0 ~len:pos
          , sub str ~pos:(pos + 1) ~len:(length str - pos - 1) )
    | None -> None

  let subo ?(pos = 0) ?len str =
    let len = match len with Some i -> i | None -> length str - pos in
    sub str ~pos ~len
end

module Map = Map.Make (String)

(** Debug trace logging *)

type ('a, 'b) fmt = ('a, Format.formatter, unit, 'b) format4
type 'a printf = ('a, unit) fmt -> 'a
type pf = {pf: 'a. 'a printf}

let fs = Format.err_formatter
let flush = Format.pp_print_newline fs<commit_msg>[sledge] Fix trace spec parsing of functions starting with underscore<commit_after>(*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 *)

module Char = struct
  include Char

  let is_lowercase = function 'a' .. 'z' | '_' -> true | _ -> false
  let is_uppercase = function 'A' .. 'Z' -> true | _ -> false
end

module String = struct
  include StringLabels

  let is_empty str = length str = 0

  let lsplit2 str ~on =
    match index_opt str on with
    | Some pos ->
        Some
          ( sub str ~pos:0 ~len:pos
          , sub str ~pos:(pos + 1) ~len:(length str - pos - 1) )
    | None -> None

  let subo ?(pos = 0) ?len str =
    let len = match len with Some i -> i | None -> length str - pos in
    sub str ~pos ~len
end

module Map = Map.Make (String)

(** Debug trace logging *)

type ('a, 'b) fmt = ('a, Format.formatter, unit, 'b) format4
type 'a printf = ('a, unit) fmt -> 'a
type pf = {pf: 'a. 'a printf}

let fs = Format.err_formatter
let flush = Format.pp_print_newline fs
{'fraction_non_alphanumeric': 0.12020109689213894, 'fraction_numerical': 0.006398537477148081, 'mean_word_length': 2.6362126245847177, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6792132', 'n_tokens_mistral': 732, 'n_tokens_neox': 703, 'n_words': 302}