<commit_before>(**************************************************************************)
(*  Copyright © 2009 <NAME> <<EMAIL>>                   *)
(*                                                                        *)
(*  This program is free software: you can redistribute it and/or modify  *)
(*  it under the terms of the GNU Affero General Public License as        *)
(*  published by the Free Software Foundation, either version 3 of the    *)
(*  License, or (at your option) any later version, with the additional   *)
(*  exemption that compiling, linking, and/or using OpenSSL is allowed.   *)
(*                                                                        *)
(*  This program is distributed in the hope that it will be useful, but   *)
(*  WITHOUT ANY WARRANTY; without even the implied warranty of            *)
(*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU     *)
(*  Affero General Public License for more details.                       *)
(*                                                                        *)
(*  You should have received a copy of the GNU Affero General Public      *)
(*  License along with this program.  If not, see                         *)
(*  <http://www.gnu.org/licenses/>.                                       *)
(**************************************************************************)

open Benl_core
open Benl_error
open Lexing

module S = Package.Set
let p = Benl_clflags.progress

let debcheck =
  let rex = Pcre.regexp "^([^ ]+) \\(= ([^)]+)\\): FAILED$" in
  fun filename ->
    let a, b = if !Benl_clflags.quiet then ("\n", "") else ("", "\n") in
    let ic = Printf.ksprintf
      Unix.open_process_in
      "edos-debcheck -quiet -failures < %s" filename
    in
    let rec loop accu =
      begin match (try Some (input_line ic) with End_of_file -> None) with
        | None ->
            accu
        | Some line ->
            begin try
              let r = Pcre.exec ~rex line in
              loop (S.add (Package.Name.of_string (Pcre.get_substring r 1)) accu)
            with Not_found ->
              Printf.eprintf "%sW: ignored line: %s%s%!" a line b;
              loop accu
            end
      end
    in
    let result = loop S.empty in
    begin match Unix.close_process_in ic with
      | Unix.WEXITED (0|1) -> ()
      | Unix.WEXITED i ->
          Printf.eprintf
            "%sW: subprocess edos-debcheck exited with code %d%s%!" a i b
      | Unix.WSIGNALED i ->
          Printf.eprintf
            "%sW: subprocess edos-debcheck died with signal %d%s%!" a i b
      | Unix.WSTOPPED i ->
          Printf.eprintf
            "%sW: subprocess edos-debcheck stopped with signal %d%s%!" a i b
    end; result

let parse_control_file kind filename do_debcheck to_forget f accu =
  let base = Filename.basename filename in
  let debcheck_data =
    match kind with
      | `binary ->
        if do_debcheck then begin
          p "Running edos-debcheck on %s..." base;
          let result = debcheck filename in
          p "\n"; Some result
        end else None
      | `source -> None
  in
  p "Parsing %s..." base;
  let result =
    with_in_file filename begin fun ic ->
      Benl_lexer.stanza_fold to_forget begin fun name p accu ->
        f
          (Package.Name.of_string name)
          (Package.of_assoc ~debcheck_data kind p)
          accu
      end (from_channel ic) accu
    end
  in p "\n"; result

let parse_config_file filename =
  with_in_file filename begin fun ic ->
    let lexbuf = from_channel ic in
    let pos = lexbuf.lex_curr_p in
    lexbuf.lex_curr_p <- { pos with pos_fname = filename };
    try
      Benl_parser.config_file Benl_lexer.token lexbuf
    with Benl_parser.Error ->
      let pos = Lexing.lexeme_start_p lexbuf in
      raise (Parsing_error
               (pos.pos_fname,
                pos.pos_lnum,
                pos.pos_cnum-pos.pos_bol))
  end

let file_content file =
  let lines = ref "" in
  let inchan = open_in file in
  try
    while true; do
      lines := Printf.sprintf "%s%s\n" !lines (input_line inchan)
    done; ""
  with End_of_file ->
    close_in inchan;
    !lines

let dump_to_file name string =
  let outchan = open_out name in
  let () = output_string outchan string in
  close_out outchan
<commit_msg>Show a useful message when parsing fails<commit_after>(**************************************************************************)
(*  Copyright © 2009 <NAME> <<EMAIL>>                   *)
(*                                                                        *)
(*  This program is free software: you can redistribute it and/or modify  *)
(*  it under the terms of the GNU Affero General Public License as        *)
(*  published by the Free Software Foundation, either version 3 of the    *)
(*  License, or (at your option) any later version, with the additional   *)
(*  exemption that compiling, linking, and/or using OpenSSL is allowed.   *)
(*                                                                        *)
(*  This program is distributed in the hope that it will be useful, but   *)
(*  WITHOUT ANY WARRANTY; without even the implied warranty of            *)
(*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU     *)
(*  Affero General Public License for more details.                       *)
(*                                                                        *)
(*  You should have received a copy of the GNU Affero General Public      *)
(*  License along with this program.  If not, see                         *)
(*  <http://www.gnu.org/licenses/>.                                       *)
(**************************************************************************)

open Benl_core
open Benl_error
open Lexing

module S = Package.Set
let p = Benl_clflags.progress

let debcheck =
  let rex = Pcre.regexp "^([^ ]+) \\(= ([^)]+)\\): FAILED$" in
  fun filename ->
    let a, b = if !Benl_clflags.quiet then ("\n", "") else ("", "\n") in
    let ic = Printf.ksprintf
      Unix.open_process_in
      "edos-debcheck -quiet -failures < %s" filename
    in
    let rec loop accu =
      begin match (try Some (input_line ic) with End_of_file -> None) with
        | None ->
            accu
        | Some line ->
            begin try
              let r = Pcre.exec ~rex line in
              loop (S.add (Package.Name.of_string (Pcre.get_substring r 1)) accu)
            with Not_found ->
              Printf.eprintf "%sW: ignored line: %s%s%!" a line b;
              loop accu
            end
      end
    in
    let result = loop S.empty in
    begin match Unix.close_process_in ic with
      | Unix.WEXITED (0|1) -> ()
      | Unix.WEXITED i ->
          Printf.eprintf
            "%sW: subprocess edos-debcheck exited with code %d%s%!" a i b
      | Unix.WSIGNALED i ->
          Printf.eprintf
            "%sW: subprocess edos-debcheck died with signal %d%s%!" a i b
      | Unix.WSTOPPED i ->
          Printf.eprintf
            "%sW: subprocess edos-debcheck stopped with signal %d%s%!" a i b
    end; result

let parse_control_file kind filename do_debcheck to_forget f accu =
  let base = Filename.basename filename in
  let debcheck_data =
    match kind with
      | `binary ->
        if do_debcheck then begin
          p "Running edos-debcheck on %s..." base;
          let result = debcheck filename in
          p "\n"; Some result
        end else None
      | `source -> None
  in
  p "Parsing %s..." base;
  let result =
    with_in_file filename begin fun ic ->
      Benl_lexer.stanza_fold to_forget begin fun name p accu ->
        f
          (Package.Name.of_string name)
          (Package.of_assoc ~debcheck_data kind p)
          accu
      end (from_channel ic) accu
    end
  in p "\n"; result

let parse_config_file filename =
  with_in_file filename begin fun ic ->
    let lexbuf = from_channel ic in
    let pos = lexbuf.lex_curr_p in
    lexbuf.lex_curr_p <- { pos with pos_fname = filename };
    try
      Benl_parser.config_file Benl_lexer.token lexbuf
    with Benl_parser.Error ->
      let pos = Lexing.lexeme_start_p lexbuf in
      let exn = Parsing_error
        (pos.pos_fname,
         pos.pos_lnum,
         pos.pos_cnum-pos.pos_bol) in
      Printf.eprintf "E: %s\n" (string_of_exn exn);
      exit 1
  end

let file_content file =
  let lines = ref "" in
  let inchan = open_in file in
  try
    while true; do
      lines := Printf.sprintf "%s%s\n" !lines (input_line inchan)
    done; ""
  with End_of_file ->
    close_in inchan;
    !lines

let dump_to_file name string =
  let outchan = open_out name in
  let () = output_string outchan string in
  close_out outchan

{'fraction_non_alphanumeric': 0.12140059894033632, 'fraction_numerical': 0.0019580741764570375, 'mean_word_length': 1.8618984838497035, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3577198', 'n_tokens_mistral': 2579, 'n_tokens_neox': 2481, 'n_words': 863}