<commit_before>class ParrotTest::Harness::Loader {
    has $!os;
    has @!files;

    method max_filename_length() {
        my $max := 0;
        for @!files {
            my $file := $_;
            my $length := pir::length__iS($file);
            if $length > $max {
                $max := $length;
            }
        }
        return $max;
    }

    method prepare_new_run() {
        @!files := [];
    }

    method get_dir_contents($path, $recurse, @contents) {
        if ! pir::defined($!os) {
            pir::loadlib("os");
            $!os := pir::new__PS('OS');
        }

        my $STAT_ISREG := 0x8000;
        my $STAT_ISDIR := 0x4000;
        my @contents_raw := $!os.readdir: ~$path;

        for @contents_raw {
            my $entry := "$path/$_";
            my @stat := $!os.stat($entry);
            my $isdir := pir::band__III(@stat[2], $STAT_ISDIR);
            my $isfile := pir::band__III(@stat[2], $STAT_ISREG);
            next if $_[0] eq ".";
            if $isfile == $STAT_ISREG {
                @contents.push($entry);
            }

            if $recurse && $isdir == $STAT_ISDIR {
                self.get_dir_contents($entry, $recurse, @contents);
            }
        }

        # TODO: Maybe read the shebang preamble to make sure we have the
        #       correct type of file.

    }

    method get_tests_from_dirs(@dirs, $recurse) {
        my @tests := < >;
        for @dirs {
            my $dir := $_;
            my @rawfiles := [];
            self.get_dir_contents($dir, $recurse, @rawfiles);
            for @rawfiles {
                my $filename := $_;
                # TODO: Break these out into a list of include and exclude
                #       patterns
                next if pir::index__ISS($filename, ".t") == -1;
                next if pir::index__ISS($filename, ".OLD") != -1;

                my $testobj := self.make_test_obj();
                $testobj.filename: $filename;
                @tests.push($testobj);
                @!files.push($filename);
            }
        }
        return @tests;
    }

    method get_tests_from_files(@filenames) {
        my @tests := [];
        for @filenames {
            my $filename := $_;
            my $testobj := self.make_test_obj();
            $testobj.filename($filename);
            @tests.push($testobj);
            @!files.push($filename);
        }
        return @tests;
    }


    method make_test_obj() {
        pir::die("Must use a subclass");<commit_msg>refactor test to see whether a file is a valid test file. Now we can subclass just that portion<commit_after>class ParrotTest::Harness::Loader {
    has $!os;
    has @!files;

    method max_filename_length() {
        my $max := 0;
        for @!files {
            my $file := $_;
            my $length := pir::length__iS($file);
            if $length > $max {
                $max := $length;
            }
        }
        return $max;
    }

    method prepare_new_run() {
        @!files := [];
    }

    method get_dir_contents($path, $recurse, @contents) {
        if ! pir::defined($!os) {
            pir::loadlib("os");
            $!os := pir::new__PS('OS');
        }

        my $STAT_ISREG := 0x8000;
        my $STAT_ISDIR := 0x4000;
        my @contents_raw := $!os.readdir: ~$path;

        for @contents_raw {
            my $entry := "$path/$_";
            my @stat := $!os.stat($entry);
            my $isdir := pir::band__III(@stat[2], $STAT_ISDIR);
            my $isfile := pir::band__III(@stat[2], $STAT_ISREG);
            next if $_[0] eq ".";
            if $isfile == $STAT_ISREG && self.is_test($entry) {
                @contents.push($entry);
            }

            if $recurse && $isdir == $STAT_ISDIR {
                self.get_dir_contents($entry, $recurse, @contents);
            }
        }

        # TODO: Maybe read the shebang preamble to make sure we have the
        #       correct type of file.

    }

    # Determine if the file is a valid test. Probably best to override this in
    # a subclass if you need different behaviors.
    method is_test($filename) {
        if (pir::index__ISS($filename, ".t") == -1) { return 0; }
        if (pir::index__ISS($filename, ".OLD") != -1) { return 0; }
        return 1;
    }

    method get_tests_from_dirs(@dirs, $recurse) {
        my @tests := < >;
        for @dirs {
            my $dir := $_;
            my @rawfiles := [];
            self.get_dir_contents($dir, $recurse, @rawfiles);
            for @rawfiles {
                my $filename := $_;
                my $testobj := self.make_test_obj();
                $testobj.filename: $filename;
                @tests.push($testobj);
                @!files.push($filename);
            }
        }
        return @tests;
    }

    method get_tests_from_files(@filenames) {
        my @tests := [];
        for @filenames {
            my $filename := $_;
            my $testobj := self.make_test_obj();
            $testobj.filename($filename);
            @tests.push($testobj);
            @!files.push($filename);
        }
        return @tests;
    }


    method make_test_obj() {
        pir::die("Must use a subclass");
{'fraction_non_alphanumeric': 0.14381849644299174, 'fraction_numerical': 0.006729475100942127, 'mean_word_length': 1.5575221238938053, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2846162', 'n_tokens_mistral': 1647, 'n_tokens_neox': 1567, 'n_words': 363}