<commit_before>module UnlabeledBytes = Bytes
open StdLabels

module Bytes = struct
  include StdLabels.Bytes

  (* [blit_string] was forgotten from the labeled version in OCaml
     4.02—4.04. *)
  let blit_string ~src ~src_pos ~dst ~dst_pos ~len =
    UnlabeledBytes.blit_string src src_pos dst dst_pos len
end

module A = Parser_automaton_internal

module Atom = struct
 type t = Sexp_ast.atom = A of string [@@unboxed]

 let is_valid str =
   let len = String.length str in
   len = 0 ||
   let rec loop ix =
     match str.[ix] with
     | '"' | '(' | ')' | ';' | '\\' -> true
     | '|' -> ix > 0 && let next = ix - 1 in str.[next] = '#' || loop next
     | '#' -> ix > 0 && let next = ix - 1 in str.[next] = '|' || loop next
     | '\000' .. '\032' | '\127' .. '\255' -> true
     | _ -> ix > 0 && loop (ix - 1)
   in
   not (loop (len - 1))

 (* XXX eventually we want to report a nice error message to the user
     at the point the conversion is made. *)
  let of_string s =
    if is_valid s then A s
    else invalid_arg(Printf.sprintf "Usexp.Atom.of_string: %S" s)

  let of_int i = A (string_of_int i)
  let of_float x = A (string_of_float x)
  let of_bool x = A (string_of_bool x)
  let of_int64 i = A (Int64.to_string i)
  let of_digest d = A (Digest.to_hex d)

  let to_string (A s) = s
end

type t =
  | Atom of Atom.t
  | Quoted_string of string
  | List of t list

type sexp = t<commit_msg>Fix Atom.is_valid<commit_after>module UnlabeledBytes = Bytes
open StdLabels

module Bytes = struct
  include StdLabels.Bytes

  (* [blit_string] was forgotten from the labeled version in OCaml
     4.02—4.04. *)
  let blit_string ~src ~src_pos ~dst ~dst_pos ~len =
    UnlabeledBytes.blit_string src src_pos dst dst_pos len
end

module A = Parser_automaton_internal

module Atom = struct
 type t = Sexp_ast.atom = A of string [@@unboxed]

 let is_valid str =
   let len = String.length str in
   len > 0 &&
   let rec loop ix =
     match str.[ix] with
     | '"' | '(' | ')' | ';' | '\\' -> true
     | '|' -> ix > 0 && let next = ix - 1 in str.[next] = '#' || loop next
     | '#' -> ix > 0 && let next = ix - 1 in str.[next] = '|' || loop next
     | '\000' .. '\032' | '\127' .. '\255' -> true
     | _ -> ix > 0 && loop (ix - 1)
   in
   not (loop (len - 1))

 (* XXX eventually we want to report a nice error message to the user
     at the point the conversion is made. *)
  let of_string s =
    if is_valid s then A s
    else invalid_arg(Printf.sprintf "Usexp.Atom.of_string: %S" s)

  let of_int i = A (string_of_int i)
  let of_float x = A (string_of_float x)
  let of_bool x = A (string_of_bool x)
  let of_int64 i = A (Int64.to_string i)
  let of_digest d = A (Digest.to_hex d)

  let to_string (A s) = s
end

type t =
  | Atom of Atom.t
  | Quoted_string of string
  | List of t list

type sexp = t
{'fraction_non_alphanumeric': 0.12792345854004253, 'fraction_numerical': 0.021261516654854713, 'mean_word_length': 2.5420326223337515, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '188846', 'n_tokens_mistral': 1115, 'n_tokens_neox': 1054, 'n_words': 370}