<commit_before>open Printf

let scriptName = "check_test"

let testOutputExt = "test_output"
let reportOutputExt = "testreport"

type usageError =
  | InvalidArguments
  | CompilerFailed of int

let failWith error =
  let errorCode, msg =
    match error with
      | InvalidArguments -> 1,
          sprintf "Expecting %s zomp-file" scriptName
      | CompilerFailed n -> 2, sprintf "Compilation failed with %d" n
  in
  fprintf stderr "%s\n" msg;
  flush stderr;
  exit errorCode

module Utils =
struct
  let withOpenFileIn fileName f =
    let stream = open_in fileName in
    try
      f stream;
      close_in stream
    with exn ->
      close_in stream;
      raise exn

  let withOpenFileOut fileName f =
    let stream = open_out fileName in
    try
      f stream;
      close_out stream
    with exn ->
      close_out stream;
      raise exn

  let printFile filename =
    let rec printStream s =
      let line = input_line s in
      print_string line;
      print_newline();
      printStream s
    in
    try
      withOpenFileIn filename printStream
    with End_of_file ->
      ()

  let forEachLineInFile fileName f =
    let rec read lineNum stream =
      try
        let line = input_line stream in
        let () = f lineNum line in
        read (lineNum+1) stream
      with End_of_file -> ()
    in
    withOpenFileIn fileName (read 1)

  let replaceExtension fileName ext =
    let noExt =
      try
        Filename.chop_extension fileName
      with Invalid_argument _ ->
        fileName
    in
    sprintf "%s.%s" noExt ext

  let safeParseInt str =
    try
      int_of_string str
    with Failure _ ->
      -1

  let (=~) str re = Str.string_match (Str.regexp re) str 0

  let stringToList str =
    let strLength = String.length str in
    let rec loop lst i =
      if i >= strLength then lst
      else (String.unsafe_get str i) :: loop lst (i+1)
    in
    loop [] 0

  let escapeHtmlText str =
    let replaceChar chr =
      let between min max = (chr >= min) && (chr <= max) in
      let isin chars = try ignore (String.index chars chr); true with Not_found -> false in
      if chr = ' ' then
        "&nbsp;"
      else if between 'a' 'z' || between 'A' 'Z' || between '0' '9' || isin ".,_-$~()" then
        String.make 1 chr
      else
        let ascii = int_of_char chr in
        sprintf "&#%d;" ascii
    in
    String.concat "" (List.map replaceChar (stringToList str))
end

open Utils

module Expectation =
struct
  type t = CompilerError | CompilerWarning | CompilerInfo | RuntimePrint
  let compilerErrorCommand = "error"
  let compilerWarningCommand = "warning"
  let compilerInfoCommand = "info"
  let runtimePrintCommand = "print"

  let parse str =
    if str = compilerErrorCommand then CompilerError
    else if str = compilerWarningCommand then CompilerWarning
    else if str = compilerInfoCommand then CompilerInfo
    else if str = runtimePrintCommand then RuntimePrint
    else (failwith "Expectation.parse")

  let verbalConcat verbForLast list =
    match List.rev list with
      | [] -> ""
      | [single] -> single
      | last :: remaining ->
        sprintf "%s %s %s"
          (String.concat ", " (List.rev remaining))
          verbForLast
          last

  let validExpectationsEnumDescr =
    verbalConcat "or"
      [compilerErrorCommand; compilerWarningCommand; compilerInfoCommand;
       runtimePrintCommand]

  let verbalDescription = function
    | CompilerError -> "error"
    | CompilerWarning -> "warning"
    | CompilerInfo -> "info"
    | RuntimePrint -> "to print line"
end

let writeHtmlHeader outFile zompFileName =
  fprintf outFile "<html>\n";
  fprintf outFile "  <head>\n";
  fprintf outFile "    <title>Report for %s</title>\n" zompFileName;
  fprintf outFile "    <style type=\"text/css\">";
  fprintf outFile "      .ok { color: green; }";
  fprintf outFile "      .failed { color: red; }";
  fprintf outFile "    </style>";
  fprintf outFile "  </head>\n";
  fprintf outFile "  <body>\n"<commit_msg>Moved function where it belongs.<commit_after>open Printf

let scriptName = "check_test"

let testOutputExt = "test_output"
let reportOutputExt = "testreport"

type usageError =
  | InvalidArguments
  | CompilerFailed of int

let failWith error =
  let errorCode, msg =
    match error with
      | InvalidArguments -> 1,
          sprintf "Expecting %s zomp-file" scriptName
      | CompilerFailed n -> 2, sprintf "Compilation failed with %d" n
  in
  fprintf stderr "%s\n" msg;
  flush stderr;
  exit errorCode

module Utils =
struct
  let withOpenFileIn fileName f =
    let stream = open_in fileName in
    try
      f stream;
      close_in stream
    with exn ->
      close_in stream;
      raise exn

  let withOpenFileOut fileName f =
    let stream = open_out fileName in
    try
      f stream;
      close_out stream
    with exn ->
      close_out stream;
      raise exn

  let printFile filename =
    let rec printStream s =
      let line = input_line s in
      print_string line;
      print_newline();
      printStream s
    in
    try
      withOpenFileIn filename printStream
    with End_of_file ->
      ()

  let forEachLineInFile fileName f =
    let rec read lineNum stream =
      try
        let line = input_line stream in
        let () = f lineNum line in
        read (lineNum+1) stream
      with End_of_file -> ()
    in
    withOpenFileIn fileName (read 1)

  let replaceExtension fileName ext =
    let noExt =
      try
        Filename.chop_extension fileName
      with Invalid_argument _ ->
        fileName
    in
    sprintf "%s.%s" noExt ext

  let safeParseInt str =
    try
      int_of_string str
    with Failure _ ->
      -1

  let (=~) str re = Str.string_match (Str.regexp re) str 0

  let stringToList str =
    let strLength = String.length str in
    let rec loop lst i =
      if i >= strLength then lst
      else (String.unsafe_get str i) :: loop lst (i+1)
    in
    loop [] 0

  let escapeHtmlText str =
    let replaceChar chr =
      let between min max = (chr >= min) && (chr <= max) in
      let isin chars = try ignore (String.index chars chr); true with Not_found -> false in
      if chr = ' ' then
        "&nbsp;"
      else if between 'a' 'z' || between 'A' 'Z' || between '0' '9' || isin ".,_-$~()" then
        String.make 1 chr
      else
        let ascii = int_of_char chr in
        sprintf "&#%d;" ascii
    in
    String.concat "" (List.map replaceChar (stringToList str))

  let verbalConcat verbForLast list =
    match List.rev list with
      | [] -> ""
      | [single] -> single
      | last :: remaining ->
        sprintf "%s %s %s"
          (String.concat ", " (List.rev remaining))
          verbForLast
          last
end

open Utils

module Expectation =
struct
  type t = CompilerError | CompilerWarning | CompilerInfo | RuntimePrint
  let compilerErrorCommand = "error"
  let compilerWarningCommand = "warning"
  let compilerInfoCommand = "info"
  let runtimePrintCommand = "print"

  let parse str =
    if str = compilerErrorCommand then CompilerError
    else if str = compilerWarningCommand then CompilerWarning
    else if str = compilerInfoCommand then CompilerInfo
    else if str = runtimePrintCommand then RuntimePrint
    else (failwith "Expectation.parse")

  let validExpectationsEnumDescr =
    verbalConcat "or"
      [compilerErrorCommand; compilerWarningCommand; compilerInfoCommand;
       runtimePrintCommand]

  let verbalDescription = function
    | CompilerError -> "error"
    | CompilerWarning -> "warning"
    | CompilerInfo -> "info"
    | RuntimePrint -> "to print line"
end

let writeHtmlHeader outFile zompFileName =
  fprintf outFile "<html>\n";
  fprintf outFile "  <head>\n";
  fprintf outFile "    <title>Report for %s</title>\n" zompFileName;
  fprintf outFile "    <style type=\"text/css\">";
  fprintf outFile "      .ok { color: green; }";
  fprintf outFile "      .failed { color: red; }";
  fprintf outFile "    </style>";
  fprintf outFile "  </head>\n";
  fprintf outFile "  <body>\n"
{'fraction_non_alphanumeric': 0.08594237245852564, 'fraction_numerical': 0.0027441686416365225, 'mean_word_length': 2.477016478751084, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '203711', 'n_tokens_mistral': 2442, 'n_tokens_neox': 2429, 'n_words': 819}