<commit_before>(*  Title:      Pure/ML-Systems/polyml-5.0.ML
    ID:         $Id$

Compatibility wrapper for Poly/ML 5.0.
*)

use "ML-Systems/polyml-4.1.4-patch.ML";
use "ML-Systems/polyml.ML";

val pointer_eq = PolyML.pointerEq;


(* improved versions of use_text/file *)
(*
local

fun use_ml name (print, err) verbose txt =
  let
    val in_buffer = ref (explode txt);
    val out_buffer = ref ([]: string list);
    fun output () = implode (rev (case ! out_buffer of "\n" :: cs => cs | cs => cs));

    val line_no = ref 1;
    fun line () = ! line_no;
    fun get () =
      (case ! in_buffer of
        [] => ""
      | c :: cs => (in_buffer := cs; if c = "\n" then line_no := ! line_no + 1 else (); c));
    fun put s = out_buffer := s :: ! out_buffer;

    fun exec () =
      (case ! in_buffer of
        [] => ()
      | _ => (PolyML.compilerEx (get, put, line, name) (); exec ()));
  in
    exec () handle exn => (err (output ()); raise exn);
    if verbose then print (output ()) else ()
  end;

in

fun use_text output = use_ml "ML text" output;

fun use_file output verbose name =
  let
    val instream = TextIO.openIn name;
    val txt = TextIO.inputAll instream before TextIO.closeIn instream;
  in use_ml name output verbose txt end;

end;
*)<commit_msg>activated improved use_ml, which captures output and reports source positions; define use_file in terms of use_ml;<commit_after>(*  Title:      Pure/ML-Systems/polyml-5.0.ML
    ID:         $Id$

Compatibility wrapper for Poly/ML 5.0.
*)

use "ML-Systems/polyml-4.1.4-patch.ML";
use "ML-Systems/polyml.ML";

val pointer_eq = PolyML.pointerEq;


(* improved versions of use_text/file *)

local

fun std_output s = (TextIO.output (TextIO.stdOut, s); TextIO.flushOut TextIO.stdOut);

fun use_ml name (print, err) verbose txt =
  let
    val in_buffer = ref (explode txt);
    val out_buffer = ref ([]: string list);
    fun output () = implode (rev (case ! out_buffer of "\n" :: cs => cs | cs => cs));

    val line_no = ref 1;
    fun line () = ! line_no;
    fun get () =
      (case ! in_buffer of
        [] => ""
      | c :: cs => (in_buffer := cs; if c = "\n" then line_no := ! line_no + 1 else (); c));
    val put_fn = ref (fn s => out_buffer := s :: ! out_buffer);

    fun exec () =
      (case ! in_buffer of
        [] => ()
      | _ => (PolyML.compilerEx (get, fn s => ! put_fn s, line, name) (); exec ()));
  in
    exec () handle exn => (err (output ()); raise exn);
    if verbose then print (output ()) else ();
    put_fn := std_output   (*run-time output, e.g. PolyML.print*)
  end;

in

fun use_text output = use_ml "ML text" output;

fun use_file output verbose name =
  let
    val instream = TextIO.openIn name;
    val txt = TextIO.inputAll instream before TextIO.closeIn instream;
  in use_ml name output verbose txt end;

end;
{'fraction_non_alphanumeric': 0.1467044649184975, 'fraction_numerical': 0.0063784549964564135, 'mean_word_length': 2.685378590078329, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4718853', 'n_tokens_mistral': 1019, 'n_tokens_neox': 987, 'n_words': 318}