<commit_before>
(** N.B. when I say "[x] is a convenience function around [y]", that just means [x] can be
    thought of in terms of [y].  In fact, [x] may not be implemented on top of [y], because
    in many cases (e.g., find/find_all) the convenience functions assume certain defaults
    that make it more efficient to drop down into C directly.
*)

open Core_kernel.Std

type t

type 'a without_trailing_none = 'a [@@deriving sexp_of]

let without_trailing_none = Fn.id

module Options = Options

external cre2__init : unit -> unit = "mlre2__init"
external cre2__create_re : Options.enum list -> string -> t = "mlre2__create_re"
external cre2__num_submatches : t -> int = "mlre2__num_submatches" [@@noalloc]

external cre2__submatch_index : t -> string -> int = "mlre2__submatch_index" [@@noalloc]

external cre2__pattern : t -> string = "mlre2__pattern"

external cre2__iter_next :
  t
  -> int
  -> int
  -> string
  -> (int * (int * int) option array option)
  = "mlre2__iter_next"

external cre2__matches : t -> string -> bool = "mlre2__matches" [@@noalloc]
external cre2__find_all : t -> int -> string -> string list = "mlre2__find_all"
external cre2__find_first : t -> int -> string -> string = "mlre2__find_first"
external cre2__rewrite_exn : t -> string -> string -> string = "mlre2__rewrite_exn"
external cre2__valid_rewrite_template : t -> string -> bool =
  "mlre2__valid_rewrite_template" [@@noalloc]
external cre2__escape : string -> string = "mlre2__escape"

type regex = t

module Exceptions = struct
  exception Regex_no_such_subpattern of int * int
  (** [Regex_no_such_subpattern (n, max)] means [n] was requested but only [max]
      subpatterns are defined (so [max] - 1 is the highest valid index) *)
  exception Regex_no_such_named_subpattern of string * string
  (** [Regex_no_such_named_subpattern (name, pattern)] *)
  exception Regex_match_failed of string
  (** [Match_failed pattern] *)
  exception Regex_submatch_did_not_capture of string * int
  (** [Regex_submatch_did_not_capture (s, i)] means the [i]th subpattern in the
      regex compiled from [s] did not capture a substring. *)
  exception Regex_compile_failed of string
  (** the string is the C library's error message, generally in the form of
      "(human-readable error): (piece of pattern that did not compile)" *)
  exception Regex_rewrite_template_invalid of string * string
  (** [Regex_rewrite_template_invalid (template, error_msg)] *)

  let _ = (* register exceptions *)
    Callback.register_exception "mlre2__Regex_no_such_subpattern"
      (Regex_no_such_subpattern (-1, -1));
    Callback.register_exception "mlre2__Regex_no_such_named_subpattern"
      (Regex_no_such_named_subpattern ("foo", "bar"));
    Callback.register_exception "mlre2__Regex_match_failed"
      (Regex_match_failed "");
    Callback.register_exception "mlre2__Regex_submatch_did_not_capture"
      (Regex_submatch_did_not_capture ("", 0));
    Callback.register_exception "mlre2__Regex_compile_failed"
      (Regex_compile_failed "");
    Callback.register_exception "mlre2__Regex_rewrite_template_invalid"
      (Regex_rewrite_template_invalid ("", ""));
  ;;
end
include Exceptions

let () = cre2__init ()    (* register custom operations *)

let create_exn ?(options=[]) pat =
  cre2__create_re (List.map ~f:Options.enum_of_t options) pat
;;

let create ?options pat = Or_error.try_with (fun () -> create_exn ?options pat)

let num_submatches t          = cre2__num_submatches t
let pattern t                 = cre2__pattern t
(*let submatch_index_exn t name =
  let i = cre2__submatch_index t name in
  if i < 0 then raise (Regex_no_such_named_subpattern (name, pattern t)) else i
let submatch_index t name     = Or_error.try_with (fun () -> submatch_index_exn t name)*)

let of_string pat = create_exn pat
let to_string t   = cre2__pattern t

module T = struct
  type nonrec t = t
  let of_string = of_string
  let to_string = to_string
end

include Binable. Of_stringable (T)
include Sexpable.Of_stringable (T)

let compare t1 t2 = String.compare (to_string t1) (to_string t2)

type id_t = [ `Index of int | `Name of string ]

let index_of_id_exn t = function
  | `Index i ->
    let max = num_submatches t in
    if i <= max then i else raise (Regex_no_such_subpattern (i, max))
  | `Name name -> cre2__submatch_index t name
;;

module Match = struct
  type t = {
    rex      : regex sexp_opaque;
    input    : string;
    captures : (int * int) option array;
  } [@@deriving sexp_of]

  let get_pos_exn ~sub t =
    let i = index_of_id_exn t.rex sub in
    let maybe_captures =
      try Array.get t.captures i with
      | Invalid_argument "index out of bounds" ->
        raise (Regex_no_such_subpattern (i, Array.length t.captures))
      | e -> raise e
    in
    match maybe_captures with
    | None        -> raise (Regex_submatch_did_not_capture (cre2__pattern t.rex, i))
    | Some retval -> retval
  ;;

  let get_exn ~sub t =
    let pos, len = get_pos_exn ~sub t in
    String.sub t.input ~pos ~len
  ;;

  let get ~sub t =
    Option.map (try Array.get t.captures (index_of_id_exn t.rex sub) with _ -> None)
      ~f:(fun (pos, len) -> String.sub t.input ~pos ~len)
<commit_msg>Fix find_first segfault when given named subpattern is not in the regex<commit_after>
(** N.B. when I say "[x] is a convenience function around [y]", that just means [x] can be
    thought of in terms of [y].  In fact, [x] may not be implemented on top of [y], because
    in many cases (e.g., find/find_all) the convenience functions assume certain defaults
    that make it more efficient to drop down into C directly.
*)

open Core_kernel.Std

type t

type 'a without_trailing_none = 'a [@@deriving sexp_of]

let without_trailing_none = Fn.id

module Options = Options

external cre2__init : unit -> unit = "mlre2__init"
external cre2__create_re : Options.enum list -> string -> t = "mlre2__create_re"
external cre2__num_submatches : t -> int = "mlre2__num_submatches" [@@noalloc]

external cre2__submatch_index : t -> string -> int = "mlre2__submatch_index" [@@noalloc]

external cre2__pattern : t -> string = "mlre2__pattern"

external cre2__iter_next :
  t
  -> int
  -> int
  -> string
  -> (int * (int * int) option array option)
  = "mlre2__iter_next"

external cre2__matches : t -> string -> bool = "mlre2__matches" [@@noalloc]
external cre2__find_all : t -> int -> string -> string list = "mlre2__find_all"
external cre2__find_first : t -> int -> string -> string = "mlre2__find_first"
external cre2__rewrite_exn : t -> string -> string -> string = "mlre2__rewrite_exn"
external cre2__valid_rewrite_template : t -> string -> bool =
  "mlre2__valid_rewrite_template" [@@noalloc]
external cre2__escape : string -> string = "mlre2__escape"

type regex = t

module Exceptions = struct
  exception Regex_no_such_subpattern of int * int
  (** [Regex_no_such_subpattern (n, max)] means [n] was requested but only [max]
      subpatterns are defined (so [max] - 1 is the highest valid index) *)
  exception Regex_no_such_named_subpattern of string * string
  (** [Regex_no_such_named_subpattern (name, pattern)] *)
  exception Regex_match_failed of string
  (** [Match_failed pattern] *)
  exception Regex_submatch_did_not_capture of string * int
  (** [Regex_submatch_did_not_capture (s, i)] means the [i]th subpattern in the
      regex compiled from [s] did not capture a substring. *)
  exception Regex_compile_failed of string
  (** the string is the C library's error message, generally in the form of
      "(human-readable error): (piece of pattern that did not compile)" *)
  exception Regex_rewrite_template_invalid of string * string
  (** [Regex_rewrite_template_invalid (template, error_msg)] *)

  let _ = (* register exceptions *)
    Callback.register_exception "mlre2__Regex_no_such_subpattern"
      (Regex_no_such_subpattern (-1, -1));
    Callback.register_exception "mlre2__Regex_no_such_named_subpattern"
      (Regex_no_such_named_subpattern ("foo", "bar"));
    Callback.register_exception "mlre2__Regex_match_failed"
      (Regex_match_failed "");
    Callback.register_exception "mlre2__Regex_submatch_did_not_capture"
      (Regex_submatch_did_not_capture ("", 0));
    Callback.register_exception "mlre2__Regex_compile_failed"
      (Regex_compile_failed "");
    Callback.register_exception "mlre2__Regex_rewrite_template_invalid"
      (Regex_rewrite_template_invalid ("", ""));
  ;;
end
include Exceptions

let () = cre2__init ()    (* register custom operations *)

let create_exn ?(options=[]) pat =
  cre2__create_re (List.map ~f:Options.enum_of_t options) pat
;;

let create ?options pat = Or_error.try_with (fun () -> create_exn ?options pat)

let num_submatches t          = cre2__num_submatches t
let pattern t                 = cre2__pattern t

let of_string pat = create_exn pat
let to_string t   = cre2__pattern t

module T = struct
  type nonrec t = t
  let of_string = of_string
  let to_string = to_string
end

include Binable. Of_stringable (T)
include Sexpable.Of_stringable (T)

let compare t1 t2 = String.compare (to_string t1) (to_string t2)

type id_t = [ `Index of int | `Name of string ]

let index_of_id_exn t = function
  | `Index i ->
    let max = num_submatches t in
    if i <= max then i else raise (Regex_no_such_subpattern (i, max))
  | `Name name ->
    let i = cre2__submatch_index t name in
    if i < 0 then raise (Regex_no_such_named_subpattern (name, pattern t)) else i
;;

module Match = struct
  type t = {
    rex      : regex sexp_opaque;
    input    : string;
    captures : (int * int) option array;
  } [@@deriving sexp_of]

  let get_pos_exn ~sub t =
    let i = index_of_id_exn t.rex sub in
    let maybe_captures =
      try Array.get t.captures i with
      | Invalid_argument "index out of bounds" ->
        raise (Regex_no_such_subpattern (i, Array.length t.captures))
      | e -> raise e
    in
    match maybe_captures with
    | None        -> raise (Regex_submatch_did_not_capture (cre2__pattern t.rex, i))
    | Some retval -> retval
  ;;

  let get_exn ~sub t =
    let pos, len = get_pos_exn ~sub t in
    String.sub t.input ~pos ~len
  ;;

  let get ~sub t =
    Option.map (try Array.get t.captures (index_of_id_exn t.rex sub) with _ -> None)
      ~f:(fun (pos, len) -> String.sub t.input ~pos ~len)

{'fraction_non_alphanumeric': 0.09754911495817935, 'fraction_numerical': 0.00904493289243338, 'mean_word_length': 3.805140186915888, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '876499', 'n_tokens_mistral': 3469, 'n_tokens_neox': 3417, 'n_words': 1039}