<commit_before>(* Run the regression tests. *)

let

    fun runTests (dirName, expect) =
    let
        (* Run a file.  Returns true if it succeeds, false if it fails. *)
        fun runTest fileName =
        let
            open PolyML.Compiler
            (* First in list is the name with no suffix. *)
            val inStream = TextIO.getInstream(TextIO.openIn fileName)
            val stream = ref inStream

            val lineNo   = ref 1;
            fun getChar () : char option =
                case TextIO.StreamIO.input1 (! stream) of
                    NONE => NONE
                |   SOME (eoln as #"\n", strm) =>
                    (
                        lineNo := !lineNo + 1;
                        stream := strm;
                        SOME eoln
                    )
                |   SOME(c, strm) => (stream := strm; SOME c)
            (* Create a private name space for each test otherwise declarations in one
               could affect another. *)
            fun makeSpace(globalLook, globalAll) =
            let
                open HashArray
                val table = hash 10
                infix 8 sub
                fun lookup s =
                    case table sub s of
                        NONE => globalLook s
                    |   SOME r => SOME r
                fun enter(s, v) = update(table, s, v)
                fun all () = fold (fn(s, v, l) => (s, v) :: l) (globalAll()) table
            in
                { lookup = lookup, enter = enter, all = all }
            end
            val { lookupFix, lookupSig, lookupVal, lookupType, lookupFunct, lookupStruct,
                  allFix, allSig, allVal, allType, allFunct, allStruct, ...} = PolyML.globalNameSpace;
            val fixSpace = makeSpace(lookupFix, allFix)
            val sigSpace = makeSpace(lookupSig, allSig)
            val valSpace = makeSpace(lookupVal, allVal)
            val typeSpace = makeSpace(lookupType, allType)
            val funSpace = makeSpace(lookupFunct, allFunct)
            val strSpace = makeSpace(lookupStruct, allStruct)

            val localNameSpace: PolyML.NameSpace.nameSpace =
            {
                lookupFix    = #lookup fixSpace,
                lookupSig    = #lookup sigSpace,
                lookupVal    = #lookup valSpace,
                lookupType   = #lookup typeSpace,
                lookupFunct  = #lookup funSpace,
                lookupStruct = #lookup strSpace,
                enterFix     = #enter fixSpace,
                enterSig     = #enter sigSpace,
                enterVal     = #enter valSpace,
                enterType    = #enter typeSpace,
                enterFunct   = #enter funSpace,
                enterStruct  = #enter strSpace,
                allFix       = #all fixSpace,
                allSig       = #all sigSpace,
                allVal       = #all valSpace,
                allType      = #all typeSpace,
                allFunct     = #all funSpace,
                allStruct    = #all strSpace
            }

        in
            (
                while not (TextIO.StreamIO.endOfStream(!stream)) do
                let
                    fun discardOut _ = ()
                    val nameSpace = PolyML.globalNameSpace
    
                    val code = PolyML.compiler(getChar, [CPOutStream discardOut, CPNameSpace localNameSpace])
                in
                    code()
                end;
                (* Normal termination: close the stream. *)
                TextIO.StreamIO.closeIn (! stream);
                true (* Succeeded. *)
            ) handle exn => (TextIO.StreamIO.closeIn(!stream); false)

        end;

        open OS.FileSys OS.Path
        val dir = openDir dirName
        fun runDir () =
            case readDir dir of
                NONE => () (* Finished *)
            |   SOME f =>
                if String.isSuffix "ML" f
                then
                (
                    print f; print " => ";
                    if runTest(joinDirFile{dir=dirName, file=f}) = expect
                    then print "Passed\n"
                    else print "Failed\n";
                    runDir()
                )
                else runDir()
    in
        runDir ();
        closeDir dir
    end;
in
    (* Each test in the Succeed directory should succeed and those in the Fail directory should fail. *)
    runTests("Succeed", true);
    runTests("Fail", false)
end;<commit_msg>Improved version of test runner that accumulates failed tests and returns a success/fail result.<commit_after>(* Run the regression tests. *)

let

    fun runTests (dirName, expect) =
    let
        (* Run a file.  Returns true if it succeeds, false if it fails. *)
        fun runTest fileName =
        let
            open PolyML.Compiler
            (* First in list is the name with no suffix. *)
            val inStream = TextIO.getInstream(TextIO.openIn fileName)
            val stream = ref inStream

            val lineNo   = ref 1;
            fun getChar () : char option =
                case TextIO.StreamIO.input1 (! stream) of
                    NONE => NONE
                |   SOME (eoln as #"\n", strm) =>
                    (
                        lineNo := !lineNo + 1;
                        stream := strm;
                        SOME eoln
                    )
                |   SOME(c, strm) => (stream := strm; SOME c)
            (* Create a private name space for each test otherwise declarations in one
               could affect another. *)
            fun makeSpace(globalLook, globalAll) =
            let
                open HashArray
                val table = hash 10
                infix 8 sub
                fun lookup s =
                    case table sub s of
                        NONE => globalLook s
                    |   SOME r => SOME r
                fun enter(s, v) = update(table, s, v)
                fun all () = fold (fn(s, v, l) => (s, v) :: l) (globalAll()) table
            in
                { lookup = lookup, enter = enter, all = all }
            end
            val { lookupFix, lookupSig, lookupVal, lookupType, lookupFunct, lookupStruct,
                  allFix, allSig, allVal, allType, allFunct, allStruct, ...} = PolyML.globalNameSpace;
            val fixSpace = makeSpace(lookupFix, allFix)
            val sigSpace = makeSpace(lookupSig, allSig)
            val valSpace = makeSpace(lookupVal, allVal)
            val typeSpace = makeSpace(lookupType, allType)
            val funSpace = makeSpace(lookupFunct, allFunct)
            val strSpace = makeSpace(lookupStruct, allStruct)

            val localNameSpace: PolyML.NameSpace.nameSpace =
            {
                lookupFix    = #lookup fixSpace,
                lookupSig    = #lookup sigSpace,
                lookupVal    = #lookup valSpace,
                lookupType   = #lookup typeSpace,
                lookupFunct  = #lookup funSpace,
                lookupStruct = #lookup strSpace,
                enterFix     = #enter fixSpace,
                enterSig     = #enter sigSpace,
                enterVal     = #enter valSpace,
                enterType    = #enter typeSpace,
                enterFunct   = #enter funSpace,
                enterStruct  = #enter strSpace,
                allFix       = #all fixSpace,
                allSig       = #all sigSpace,
                allVal       = #all valSpace,
                allType      = #all typeSpace,
                allFunct     = #all funSpace,
                allStruct    = #all strSpace
            }

        in
            (
                while not (TextIO.StreamIO.endOfStream(!stream)) do
                let
                    fun discardOut _ = ()
                    val nameSpace = PolyML.globalNameSpace
    
                    val code = PolyML.compiler(getChar, [CPOutStream discardOut, CPNameSpace localNameSpace])
                in
                    code()
                end;
                (* Normal termination: close the stream. *)
                TextIO.StreamIO.closeIn (! stream);
                true (* Succeeded. *)
            ) handle exn => (TextIO.StreamIO.closeIn(!stream); false)

        end;

        open OS.FileSys OS.Path
        val dir = openDir dirName
        fun runDir (fails: string list) =
            case readDir dir of
                NONE => fails (* Finished *)
            |   SOME f =>
                if String.isSuffix "ML" f
                then
                (
                    print f; print " => ";
                    if runTest(joinDirFile{dir=dirName, file=f}) = expect
                    then (print "Passed\n"; runDir fails)
                    else (print "Failed!!\n"; runDir(fails @ [joinDirFile{dir=dirName, file=f}]))
                )
                else runDir fails
        val failedTests = runDir []
    in
        closeDir dir;
        failedTests
    end;
in
    (* Each test in the Succeed directory should succeed and those in the Fail directory should fail. *)
    case runTests("Succeed", true) @ runTests("Fail", false) of
        [] => true (* All succeeded *)
    |   failedTests => (print "\nFailed Tests: "; print(String.concatWith " " failedTests); print "\n"; false)
end;
{'fraction_non_alphanumeric': 0.08056667027143939, 'fraction_numerical': 0.0012977181788688223, 'mean_word_length': 1.2452051468803107, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3347555', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2274, 'n_words': 720}