<commit_before>namespace PSql
{
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Linq;
    using System.Management.Automation;
    using System.Text;
    using System.Text.RegularExpressions;
    using System.Threading;
    using System.Threading.Tasks;

    internal static class Worker
    {
        internal const int All = -1, Any = 0;
    }

    public class Module
    {
        public const string InitModuleName = "init";

        public Module(string name)
        {
            Name     = Helpers.RequireName(name);
            Provides = new SortedSet<string>();
            Requires = new SortedSet<string>();
            Script   = new StringBuilder();    
            WorkerId = Worker.Any;

            Provides.Add(name);

            if (name != InitModuleName)
                Requires.Add(InitModuleName);
        }

        public Module(Module other, int workerId)
        {
            Name     = other.Name;
            Provides = other.Provides;
            Requires = other.Requires;
            Script   = other.Script;
            WorkerId = workerId;
        }

        public string            Name     { get; private set; }
        public SortedSet<string> Provides { get; private set; }
        public SortedSet<string> Requires { get; private set; }
        public StringBuilder     Script   { get; private set; }

        // Worker affinity:
        //   Worker.All => run on all workers
        //   Worker.Any => run on any worker
        //   some n > 0 => run on a specific worker
        public int WorkerId { get; set; }

        public bool CanRunOnWorker(int workerId)
        {
            return WorkerId == Worker.Any
                || WorkerId == workerId;
        }
    }

    public class Subject
    {
        public Subject(string name)
        {
            Name       = Helpers.RequireName(name);
            ProvidedBy = new List<Module>();
            RequiredBy = new List<Module>();
        }

        public string       Name       { get; private set; }
        public List<Module> ProvidedBy { get; private set; }
        public List<Module> RequiredBy { get; private set; }
    }

    public class ModuleRunner
    {
        private static readonly Regex EolRegex = new Regex(
            @"\r?\n",
            RegexOptions.Compiled |
            RegexOptions.CultureInvariant
        );
<commit_msg>ToString methods for debugging.<commit_after>namespace PSql
{
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Linq;
    using System.Management.Automation;
    using System.Text;
    using System.Text.RegularExpressions;
    using System.Threading;
    using System.Threading.Tasks;

    internal static class Worker
    {
        internal const int All = -1, Any = 0;
    }

    public class Module
    {
        public const string InitModuleName = "init";

        public Module(string name)
        {
            Name     = Helpers.RequireName(name);
            Provides = new SortedSet<string>();
            Requires = new SortedSet<string>();
            Script   = new StringBuilder();    
            WorkerId = Worker.Any;

            Provides.Add(name);

            if (name != InitModuleName)
                Requires.Add(InitModuleName);
        }

        public Module(Module other, int workerId)
        {
            Name     = other.Name;
            Provides = other.Provides;
            Requires = other.Requires;
            Script   = other.Script;
            WorkerId = workerId;
        }

        public string            Name     { get; private set; }
        public SortedSet<string> Provides { get; private set; }
        public SortedSet<string> Requires { get; private set; }
        public StringBuilder     Script   { get; private set; }

        // Worker affinity:
        //   Worker.All => run on all workers
        //   Worker.Any => run on any worker
        //   some n > 0 => run on a specific worker
        public int WorkerId { get; set; }

        public bool CanRunOnWorker(int workerId)
        {
            return WorkerId == Worker.Any
                || WorkerId == workerId;
        }

        public override string ToString()
        {
            return string.Format("{0} (WorkerId: {1}, Provides: {2}, Requires: {3})",
                Name, WorkerId,
                string.Join(", ", Provides),
                string.Join(", ", Requires)
            );
        }
    }

    public class Subject
    {
        public Subject(string name)
        {
            Name       = Helpers.RequireName(name);
            ProvidedBy = new List<Module>();
            RequiredBy = new List<Module>();
        }

        public string       Name       { get; private set; }
        public List<Module> ProvidedBy { get; private set; }
        public List<Module> RequiredBy { get; private set; }

        public override string ToString()
        {
            return string.Format("{0} (ProvidedBy: {1}, RequiredBy: {2})",
                Name,
                string.Join(", ", ProvidedBy.Select(m => m.Name)),
                string.Join(", ", RequiredBy.Select(m => m.Name))
            );
        }
    }

    public class ModuleRunner
    {
        private static readonly Regex EolRegex = new Regex(
            @"\r?\n",
            RegexOptions.Compiled |
            RegexOptions.CultureInvariant
        );

{'fraction_non_alphanumeric': 0.09269817914290969, 'fraction_numerical': 0.002391024462019496, 'mean_word_length': 1.6069031639501439, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2088468', 'n_tokens_mistral': 1459, 'n_tokens_neox': 1411, 'n_words': 394}