<commit_before>(**********************************************************************
 *                                                                    *
 *                            OCamlFormat                             *
 *                                                                    *
 *  Copyright (c) 2017-present, Facebook, Inc.  All rights reserved.  *
 *                                                                    *
 *  This source code is licensed under the MIT license found in the   *
 *  LICENSE file in the root directory of this source tree.           *
 *                                                                    *
 **********************************************************************)
open Migrate_ast

type t = string
(** Concrete syntax. *)

let create s = s

let position_before t (pos : Lexing.position) =
  let is_closing c =
    Char.equal c ')' || Char.equal c ']' || Char.equal c '}'
  in
  let maybe_check_alphanum c =
    if Char.is_alphanum t.[pos.pos_cnum] then false else Char.is_alphanum c
  in
  let f _ c =
    maybe_check_alphanum c || Char.is_whitespace c || is_closing c
  in
  let pos_cnum_opt = String.rfindi t ~pos:pos.pos_cnum ~f in
  Option.map pos_cnum_opt ~f:(fun x -> {pos with pos_cnum= x + 1})

let string_between t (l1 : Location.t) (l2 : Location.t) =
  let pos = l1.loc_end.pos_cnum in
  let len = Position.distance l1.loc_end l2.loc_start in
  if
    len < 0
    (* can happen e.g. if comment is within a parenthesized expression *)
  then None
  else if
    String.length t < pos + len<commit_msg>Fix exception when calling String.sub (#832)<commit_after>(**********************************************************************
 *                                                                    *
 *                            OCamlFormat                             *
 *                                                                    *
 *  Copyright (c) 2017-present, Facebook, Inc.  All rights reserved.  *
 *                                                                    *
 *  This source code is licensed under the MIT license found in the   *
 *  LICENSE file in the root directory of this source tree.           *
 *                                                                    *
 **********************************************************************)
open Migrate_ast

type t = string
(** Concrete syntax. *)

let create s = s

let position_before t (pos : Lexing.position) =
  let is_closing c =
    Char.equal c ')' || Char.equal c ']' || Char.equal c '}'
  in
  let maybe_check_alphanum c =
    if Char.is_alphanum t.[pos.pos_cnum] then false else Char.is_alphanum c
  in
  let f _ c =
    maybe_check_alphanum c || Char.is_whitespace c || is_closing c
  in
  let pos_cnum_opt = String.rfindi t ~pos:pos.pos_cnum ~f in
  Option.map pos_cnum_opt ~f:(fun x -> {pos with pos_cnum= x + 1})

let string_between t (l1 : Location.t) (l2 : Location.t) =
  let pos = l1.loc_end.pos_cnum in
  let len = Position.distance l1.loc_end l2.loc_start in
  if
    len < 0 || pos < 0
    (* can happen e.g. if comment is within a parenthesized expression *)
  then None
  else if
    String.length t < pos + len
{'fraction_non_alphanumeric': 0.16126013724267, 'fraction_numerical': 0.008109794135995009, 'mean_word_length': 1.6009732360097324, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3773114', 'n_tokens_mistral': 876, 'n_tokens_neox': 832, 'n_words': 274}