<commit_before>(* Run the regression tests. *)

fun runTests parentDir =
let
    val defaultInlineSize = ! PolyML.Compiler.maxInlineSize

    fun runTests (dirName, expect) =
    let
        (* Run a file.  Returns true if it succeeds, false if it fails. *)
        fun runTest fileName =
        let
            open PolyML.Compiler
            (* Max inline size is not available as a CP parameter and some tests
               adjust it.  Set it to the default before each test. *)
            val () = maxInlineSize := defaultInlineSize (* Set it to the default *)
            (* First in list is the name with no suffix. *)
            val inStream = TextIO.getInstream(TextIO.openIn fileName)
            val stream = ref inStream

            val lineNo   = ref 1;
            fun getChar () : char option =
                case TextIO.StreamIO.input1 (! stream) of
                    NONE => NONE
                |   SOME (eoln as #"\n", strm) =>
                    (
                        lineNo := !lineNo + 1;
                        stream := strm;
                        SOME eoln
                    )
                |   SOME(c, strm) => (stream := strm; SOME c)
            (* Create a private name space for each test otherwise declarations in one
               could affect another. *)
            fun makeSpace(globalLook, globalAll) =
            let
                open HashArray
                val table = hash 10
                infix 8 sub
                fun lookup s =
                    case table sub s of
                        NONE => globalLook s
                    |   SOME r => SOME r
                fun enter(s, v) = update(table, s, v)
                fun all () = fold (fn(s, v, l) => (s, v) :: l) (globalAll()) table<commit_msg>Reset "debug" switch before each test.<commit_after>(* Run the regression tests. *)

fun runTests parentDir =
let
    val defaultInlineSize = ! PolyML.Compiler.maxInlineSize

    fun runTests (dirName, expect) =
    let
        (* Run a file.  Returns true if it succeeds, false if it fails. *)
        fun runTest fileName =
        let
            open PolyML.Compiler
            (* Max inline size is not available as a CP parameter and some tests
               adjust it.  Set it to the default before each test. *)
            val () = maxInlineSize := defaultInlineSize (* Set it to the default *)
            val () = debug := false (* Reset this *)
            (* First in list is the name with no suffix. *)
            val inStream = TextIO.getInstream(TextIO.openIn fileName)
            val stream = ref inStream

            val lineNo   = ref 1;
            fun getChar () : char option =
                case TextIO.StreamIO.input1 (! stream) of
                    NONE => NONE
                |   SOME (eoln as #"\n", strm) =>
                    (
                        lineNo := !lineNo + 1;
                        stream := strm;
                        SOME eoln
                    )
                |   SOME(c, strm) => (stream := strm; SOME c)
            (* Create a private name space for each test otherwise declarations in one
               could affect another. *)
            fun makeSpace(globalLook, globalAll) =
            let
                open HashArray
                val table = hash 10
                infix 8 sub
                fun lookup s =
                    case table sub s of
                        NONE => globalLook s
                    |   SOME r => SOME r
                fun enter(s, v) = update(table, s, v)
                fun all () = fold (fn(s, v, l) => (s, v) :: l) (globalAll()) table
{'fraction_non_alphanumeric': 0.08039647577092511, 'fraction_numerical': 0.003303964757709251, 'mean_word_length': 1.263551401869159, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4580781', 'n_tokens_mistral': 950, 'n_tokens_neox': 893, 'n_words': 359}