<commit_before>(* $Id$ *)

open StdLabels
open Printf
open Str

let camlbegin = "\\caml\n"
let camlend = "\\endcaml\n"
let camlin = "\\\\?\\1"
let camlout = "\\\\:\\1"
let camlbunderline = "\\<"
let camleunderline = "\\>"

let camllight = ref "TERM=norepeat ocaml"
let linelen = ref 72
let outfile = ref ""
let cut_at_blanks = ref false
let files = ref []

let _ =
  Arg.parse ["-n", Arg.Int (fun n -> linelen := n), "line length";
             "-o", Arg.String (fun s -> outfile := s), "output";
             "-caml", Arg.String (fun s -> camllight := s), "toplevel";
             "-w", Arg.Set cut_at_blanks, "cut at blanks"]
    (fun s -> files := s :: !files)
    "caml-tex2: "

let (~!) =
  let memo = ref [] in
  fun key ->
    try List.assq key !memo
    with Not_found ->
      let data = Str.regexp key in
      memo := (key, data) :: !memo;
      data

(** The Output module deals with the analysis and classification
    of the interpreter output and the parsing of status-related options
    or annotations for the caml_example environment *)
module Output = struct

  (** Interpreter output status *)
  type status =
    | Ok
    | Warning of int
    | Error

  type kind = Annotation | Option

  (** Pretty printer for status *)
  let pp_status ppf = function
    | Error -> Printf.fprintf ppf "error"
    | Ok -> Printf.fprintf ppf "ok"
    | Warning n -> Printf.fprintf ppf "warning %d" n

  (** Pretty printer for status preceded with an undefined determinant *)
  let pp_a_status ppf = function
    | Error -> Printf.fprintf ppf "an error"
    | Ok -> Printf.fprintf ppf "an ok"
    | Warning n -> Printf.fprintf ppf "a warning %d" n

  (** {2 Exceptions } *)
  exception Parsing_error of kind * string
  type unexpected_report = {source:string; expected:status; got:status}
  exception Unexpected_status of unexpected_report

  let print_unexpected {source; expected; got} =
    if expected = Ok then
      Printf.eprintf
        "Error when evaluating a caml_example environment in %s:\n\
         unexpected %a.\n\
         If %a status was expected, add an [@@expect %a] annotation.\n"
        source
        pp_status got
        pp_a_status got
        pp_status got
    else
      Printf.eprintf
        "Error when evaluating a guarded caml_example environment in %s:\n \
         %a status was expected, got %a status.\n\
         If %a states was in fact expected, change the status annotation to\
         [@@expect %a].\n"
        source
        pp_a_status expected
        pp_a_status got
        pp_a_status got
        pp_status got;
    flush stderr

  let print_parsing_error k s =
    match k with
    | Option ->
        Printf.eprintf
          "Unknown caml_example option: [%s].\n\
           Supported options are \"ok\",\"error\", or \"warning=n\" (with n \
           a warning number).\n" s
    | Annotation ->
        Printf.eprintf
          "Unknown caml_example phrase annotation: [@@expect %s].\n\
           Supported annotations are [@@expect ok], [@@expect error],\n\
           and [@@expect warning n] (with n a warning number).\n" s


  (** {2 Output analysis} *)
  let catch_error s =
    if string_match ~!{|Error:|} s 0 then Some Error else None

  let catch_warning s =
    if string_match ~!{|Warning \([0-9]+\):|} s 0 then
      Some (Warning (int_of_string @@ matched_group 1 s))
    else
      None

  let status s = match catch_warning s, catch_error s with
    | Some w, _ -> w
    | _, Some e -> e
    | None, None -> Ok

  (** {2 Parsing caml_example options } *)

  (** Parse [warning=n] options for caml_example options *)
  let parse_warning s =
    if string_match ~!{|warning=\([0-9]+\)|} s 0 then
      Some (Warning (int_of_string @@ matched_group 1 s))
    else
      None

  (** Parse [warning n] annotations *)
  let parse_local_warning s =
    if string_match ~!{|warning \([0-9]+\)|} s 0 then
      Some (Warning (int_of_string @@ matched_group 1 s))
    else
      None

  let parse_error s =
    if s="error" then Some Error else None

  let parse_ok s =
    if s = "ok" then Some Ok else None

  (** Parse the environment-wide expected status output *)
  let expected s =
    match parse_warning s, parse_error s with
    | Some w, _ -> w
    | _, Some e -> e
    | None, None -> raise (Parsing_error (Option,s))

  (** Parse the local (i.e. phrase-wide) expected status output *)
  let local_expected s =
    match parse_local_warning s, parse_error s, parse_ok s with
    | Some w, _, _ -> w
    | _, Some e, _ -> e
    | _, _, Some ok -> ok
    | None, None, None -> raise (Parsing_error (Annotation,s))

end

let caml_input, caml_output =
  let cmd = !camllight ^ " 2>&1" in
  try Unix.open_process cmd with _ -> failwith "Cannot start toplevel"
let () =
  at_exit (fun () -> ignore (Unix.close_process (caml_input, caml_output)));
  ignore (input_line caml_input);
  ignore (input_line caml_input)

let read_output () =
  let input = ref (input_line caml_input) in
  input := replace_first ~!"^# *" "" !input;
  let underline =
    if string_match ~!"Characters *\\([0-9]+\\)-\\([0-9]+\\):$" !input 0
    then
      let b = int_of_string (matched_group 1 !input)
      and e = int_of_string (matched_group 2 !input) in
      input := input_line caml_input;
      b, e
    else 0, 0
  in
  let output = Buffer.create 256 in
  while not (string_match ~!".*\"end_of_input\"$" !input 0) do
    prerr_endline !input;
    Buffer.add_string output !input;<commit_msg>Fix caml_tex2 output pattern matrix<commit_after>(* $Id$ *)

open StdLabels
open Printf
open Str

let camlbegin = "\\caml\n"
let camlend = "\\endcaml\n"
let camlin = "\\\\?\\1"
let camlout = "\\\\:\\1"
let camlbunderline = "\\<"
let camleunderline = "\\>"

let camllight = ref "TERM=norepeat ocaml"
let linelen = ref 72
let outfile = ref ""
let cut_at_blanks = ref false
let files = ref []

let _ =
  Arg.parse ["-n", Arg.Int (fun n -> linelen := n), "line length";
             "-o", Arg.String (fun s -> outfile := s), "output";
             "-caml", Arg.String (fun s -> camllight := s), "toplevel";
             "-w", Arg.Set cut_at_blanks, "cut at blanks"]
    (fun s -> files := s :: !files)
    "caml-tex2: "

let (~!) =
  let memo = ref [] in
  fun key ->
    try List.assq key !memo
    with Not_found ->
      let data = Str.regexp key in
      memo := (key, data) :: !memo;
      data

(** The Output module deals with the analysis and classification
    of the interpreter output and the parsing of status-related options
    or annotations for the caml_example environment *)
module Output = struct

  (** Interpreter output status *)
  type status =
    | Ok
    | Warning of int
    | Error

  type kind = Annotation | Option

  (** Pretty printer for status *)
  let pp_status ppf = function
    | Error -> Printf.fprintf ppf "error"
    | Ok -> Printf.fprintf ppf "ok"
    | Warning n -> Printf.fprintf ppf "warning %d" n

  (** Pretty printer for status preceded with an undefined determinant *)
  let pp_a_status ppf = function
    | Error -> Printf.fprintf ppf "an error"
    | Ok -> Printf.fprintf ppf "an ok"
    | Warning n -> Printf.fprintf ppf "a warning %d" n

  (** {2 Exceptions } *)
  exception Parsing_error of kind * string
  type unexpected_report = {source:string; expected:status; got:status}
  exception Unexpected_status of unexpected_report

  let print_unexpected {source; expected; got} =
    if expected = Ok then
      Printf.eprintf
        "Error when evaluating a caml_example environment in %s:\n\
         unexpected %a.\n\
         If %a status was expected, add an [@@expect %a] annotation.\n"
        source
        pp_status got
        pp_a_status got
        pp_status got
    else
      Printf.eprintf
        "Error when evaluating a guarded caml_example environment in %s:\n \
         %a status was expected, got %a status.\n\
         If %a states was in fact expected, change the status annotation to\
         [@@expect %a].\n"
        source
        pp_a_status expected
        pp_a_status got
        pp_a_status got
        pp_status got;
    flush stderr

  let print_parsing_error k s =
    match k with
    | Option ->
        Printf.eprintf
          "Unknown caml_example option: [%s].\n\
           Supported options are \"ok\",\"error\", or \"warning=n\" (with n \
           a warning number).\n" s
    | Annotation ->
        Printf.eprintf
          "Unknown caml_example phrase annotation: [@@expect %s].\n\
           Supported annotations are [@@expect ok], [@@expect error],\n\
           and [@@expect warning n] (with n a warning number).\n" s


  (** {2 Output analysis} *)
  let catch_error s =
    if string_match ~!{|Error:|} s 0 then Some Error else None

  let catch_warning s =
    if string_match ~!{|Warning \([0-9]+\):|} s 0 then
      Some (Warning (int_of_string @@ matched_group 1 s))
    else
      None

  let status s = match catch_warning s, catch_error s with
    | Some w, _ -> w
    | None, Some e -> e
    | None, None -> Ok

  (** {2 Parsing caml_example options } *)

  (** Parse [warning=n] options for caml_example options *)
  let parse_warning s =
    if string_match ~!{|warning=\([0-9]+\)|} s 0 then
      Some (Warning (int_of_string @@ matched_group 1 s))
    else
      None

  (** Parse [warning n] annotations *)
  let parse_local_warning s =
    if string_match ~!{|warning \([0-9]+\)|} s 0 then
      Some (Warning (int_of_string @@ matched_group 1 s))
    else
      None

  let parse_error s =
    if s="error" then Some Error else None

  let parse_ok s =
    if s = "ok" then Some Ok else None

  (** Parse the environment-wide expected status output *)
  let expected s =
    match parse_warning s, parse_error s with
    | Some w, _ -> w
    | None, Some e -> e
    | None, None -> raise (Parsing_error (Option,s))

  (** Parse the local (i.e. phrase-wide) expected status output *)
  let local_expected s =
    match parse_local_warning s, parse_error s, parse_ok s with
    | Some w, _, _ -> w
    | None, Some e, _ -> e
    | None, None, Some ok -> ok
    | None, None, None -> raise (Parsing_error (Annotation,s))

end

let caml_input, caml_output =
  let cmd = !camllight ^ " 2>&1" in
  try Unix.open_process cmd with _ -> failwith "Cannot start toplevel"
let () =
  at_exit (fun () -> ignore (Unix.close_process (caml_input, caml_output)));
  ignore (input_line caml_input);
  ignore (input_line caml_input)

let read_output () =
  let input = ref (input_line caml_input) in
  input := replace_first ~!"^# *" "" !input;
  let underline =
    if string_match ~!"Characters *\\([0-9]+\\)-\\([0-9]+\\):$" !input 0
    then
      let b = int_of_string (matched_group 1 !input)
      and e = int_of_string (matched_group 2 !input) in
      input := input_line caml_input;
      b, e
    else 0, 0
  in
  let output = Buffer.create 256 in
  while not (string_match ~!".*\"end_of_input\"$" !input 0) do
    prerr_endline !input;
    Buffer.add_string output !input;
{'fraction_non_alphanumeric': 0.1221988475258392, 'fraction_numerical': 0.006677032836366962, 'mean_word_length': 2.5236867547534643, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1520187', 'n_tokens_mistral': 3670, 'n_tokens_neox': 3524, 'n_words': 1240}