<commit_before>using IniParser;
using IniParser.Model;
using Svetomech.Utilities;
using System;
using System.Configuration;
using System.Globalization;
using System.IO;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading;
using static Svetomech.Utilities.NativeMethods;
using static Svetomech.Utilities.SimpleApp;
using static Svetomech.Utilities.SimplePlatform;

namespace SimpleMaid
{
  internal static partial class Program
  {
    private static bool Hidden { get; set; } = false;
    private static string State { set { Console.Title = $"{Application.ProductName}: {value}"; } }

    private static DirectoryInfo desiredAppDirectory;
    private static FileInfo mainConfigFile;
    private static IntPtr mainWindowHandle;
    private static Mutex singleInstance;

    private static Thread connectionThread;
    private static Thread commandThread;
    private static Thread chatThread;
    private static volatile bool busyConnectionWise = true;
    private static volatile bool busyCommandWise = false;
    private static volatile bool busyChatWise = false;

    private static readonly bool runningWindows = (RunningPlatform() == Platform.Windows);
    private static volatile bool internetAlive = true;

    // TODO: Get rid of these two
    private static volatile string machine;
    private static volatile string pass;

    internal static frmChatWindow ChatboxWindow = null;
    internal static volatile bool ChatboxExit = false;
    internal static volatile string SupportChatMessage;
    internal static volatile string UserChatMessage;
    internal static volatile string ChatCommand;


    private static string Set(string tag, string value)
    {
      tag = $"{Application.ProductName}_{tag}";

      var encoding = new UTF8Encoding();
      byte[] requestBody = encoding.GetBytes($"tag={tag}&value={value}&fmt=html");

      var request = (HttpWebRequest)WebRequest.Create($"{Variables.ServerAddress}/storeavalue");
      request.Method = "POST";
      request.Credentials = Variables.AccountCredentials;
      request.ContentType = "application/x-www-form-urlencoded";
      request.ContentLength = requestBody.Length;
      request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";

      try
      {
        using (var requestStream = request.GetRequestStream())
        {
          requestStream.Write(requestBody, 0, requestBody.Length);
        }
      }
      catch (WebException)
      {
        reportWebError();
        internetAlive = false;
        return resources.WebErrorMessage;
      }
      try { using (var response = request.GetResponse()) ;}
      catch (WebException)
      {
        reportWebError();
        internetAlive = false;
        return resources.WebErrorMessage;
      }

      resetConsoleColor();
      Console.ForegroundColor = ConsoleColor.Gray;
      Console.WriteLine($"SET  {tag}  {value}\n");

      return String.Empty;
    }

    private static string Get(string tag)
    {
      tag = $"{Application.ProductName}_{tag}";

      string value;

      var encoding = new UTF8Encoding();
      byte[] requestBody = encoding.GetBytes($"tag={tag}&fmt=html");

      var request = (HttpWebRequest)WebRequest.Create($"{Variables.ServerAddress}/getvalue");
      request.Method = "POST";
      request.Credentials = Variables.AccountCredentials;
      request.ContentType = "application/x-www-form-urlencoded";
      request.ContentLength = requestBody.Length;
      request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";

      try
      {
        using (var requestStream = request.GetRequestStream())
        {
          requestStream.Write(requestBody, 0, requestBody.Length);
        }
      }
      catch (WebException)
      {
        reportWebError();
        internetAlive = false;
        return resources.WebErrorMessage;
      }

      // TODO: Refactor
      try
      {
        using (var response = request.GetResponse())
        using (var responseStream = response.GetResponseStream())
        using (var sr = new StreamReader(responseStream))
        {
          value = sr.ReadToEnd();
          value = value.Substring(value.IndexOf(tag) + tag.Length + 4);
          value = value.Remove(value.IndexOf("\""));
        }
      }
      catch (WebException)
      {
        reportWebError();
        internetAlive = false;
        return resources.WebErrorMessage;
      }

      // TODO: Rewrite - problems with decoding
      value = decodeEncodedNonAsciiCharacters(value);
      value = value.Replace(@"\/", @"/");
      value = value.Replace(@"\\", @"\");
      value = WebUtility.HtmlDecode(value);

      resetConsoleColor();
      Console.ForegroundColor = ConsoleColor.DarkGreen;
      Console.WriteLine($"GET  {tag}  {value}\n");

      return value;
    }

    private static void SetUntilSet(string tag, string value)
    {
      while (resources.WebErrorMessage == Set(tag, value))
      {
        Thread.Sleep(Variables.GeneralDelay);
      }
    }

    private static string GetUntilGet(string tag)
    {
      string value;

      while (resources.WebErrorMessage == (value = Get(tag)))
      {
        Thread.Sleep(Variables.GeneralDelay);
      }

      return value;
    }


    // TODO: Remove regions, refactor
    [STAThread]
    private static void Main(string[] args)
    {
      System.Windows.Forms.Application.EnableVisualStyles();
      Console.Clear();

      #region Exit (if running as admin/root)
      if (IsElevated())
      {
        reportGeneralError(resources.AdminErrorMessage);
        exit();
      }
      #endregion

      desiredAppDirectory = new DirectoryInfo(Path.Combine(Environment.GetFolderPath(
        Environment.SpecialFolder.LocalApplicationData), Application.CompanyName, Application.ProductName));
      mainConfigFile = new FileInfo(Path.Combine(desiredAppDirectory.FullName,
        (Variables.ConfigName != Variables.KeywordDefault) ? Variables.ConfigName : $"{Application.ProductName}.ini"));

      #region Console arguments
      bool rogueArgFound = false;
      bool autorunArgFound = false;
      bool passArgFound = false; // <-- only need it for clarity
      bool langArgFound = false; // <-- only need it for clarity

      string passArg = Variables.DefaultPassword;
      string langArg = CultureInfo.InstalledUICulture.Name;

      if (args.Length >= 1)
      {
        int adjustedLength = args.Length;

        for (int i = 0; i < args.Length; ++i)
        {
          if (!rogueArgFound && (rogueArgFound = (args[i] == Variables.RogueArgument)))
            adjustedLength--;

          if (!autorunArgFound && (autorunArgFound = (args[i] == Variables.AutorunArgument)))
            adjustedLength--;

          if (adjustedLength >= 2)
          {
            if (!passArgFound && (args[i] == Variables.PasswordArgument))
            {
              passArg = (passArgFound = (i + 1 < adjustedLength)) ? args[i + 1] : passArg;

              if (passArgFound)
                adjustedLength -= 2;
              else
                adjustedLength--;
            }
            else if (!langArgFound && (args[i] == Variables.LanguageArgument))
            {
              langArg = (langArgFound = (i + 1 < adjustedLength)) ? args[i + 1] : langArg;

              if (langArgFound)
                adjustedLength -= 2;
              else
                adjustedLength--;
            }
          }
        }
      }
      #endregion

      #region Hide window (if autorun)
      mainWindowHandle = GetConsoleWindow();
      bool inDesiredDir = desiredAppDirectory.IsEqualTo(Application.StartupPath);
      if (inDesiredDir || rogueArgFound)
      {
        ShowWindow(mainWindowHandle, SW_HIDE);
        Program.Hidden = true;
      }
      #endregion

      // Localization
      CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.GetCultureInfo(langArg);

      // TODO: Move so it happens AFTER startup directory management
      #region Exit (if already running)
      singleInstance = new Mutex(false, "Local\\" + Application.AssemblyGuid);
      if (!singleInstance.WaitOne(0, false))
      {
        reportPastSelf();
        exit();
      }
      #endregion

      #region Startup directory management
      if (!inDesiredDir)
      {
        var svtFolder = new DirectoryInfo(ConfigurationManager.AppSettings["SvtFolderName"]);
        string[] filePaths = { Application.ExecutablePath, ConfigurationManager.OpenExeConfiguration(ConfigurationUserLevel.None).FilePath,
          Assembly.GetAssembly(typeof(FileIniDataParser)).Location };
        try
        {
          var desiredAppSubdirectory = new DirectoryInfo(Path.Combine(desiredAppDirectory.FullName, svtFolder.Name));

          svtFolder.CopyTo(desiredAppSubdirectory, false);

          foreach (var filePath in filePaths)
          {
            File.Copy(filePath, Path.Combine(desiredAppDirectory.FullName, Path.GetFileName(filePath)), true);
          }
        }
        catch //unauthorized, io, notsupported
        {
          // 1. stop other instance using guid technique
          //   if there is no other instance running, (?recursive or ?continue)
          // 2. try to copy again
          //   if unlucky, (?recursive or ?continue)
          // 3. start that instance (?with a delay through .bat file)
          // 4. exit
        }
      }
      #endregion


      var config_parser = new FileIniDataParser();
      IniData configuration;

      bool   machineConfigured = false;
      string machineName       = createMachine();
      string machinePassword   = <PASSWORD>;
      bool   autoRun           = autorunArgFound;

      bool firstRun;
      bool promptShown = false;
      if (firstRun = !mainConfigFile.Exists)
      {
        configuration = new IniData();
        configuration.Sections.AddSection("Service");


        configuration["Service"].AddKey("bMachineConfigured", machineConfigured.ToString());
        configuration["Service"].AddKey("sMachineName", machineName);

        validateMemoryPassword(ref configuration, ref machinePassword, ref promptShown);

        configuration["Service"].AddKey("bAutoRun", autoRun.ToString());
      }
      else
      {
        configuration = config_parser.ReadFile(mainConfigFile.FullName, Encoding.UTF8);


        machineConfigured = bool.Parse(configuration["Service"]["bMachineConfigured"]);

        if (isNameOK(configuration["Service"]["sMachineName"]))
        {
          machineName = configuration["Service"]["sMachineName"];
        }
        else
        {
          configuration["Service"]["sMachineName"] = machineName;
          machineConfigured = false;
        }

        if (!passArgFound)
        {
          validateConfigPassword(ref configuration, ref machinePassword, ref promptShown);
        }
        else
        {
          if (isPasswordOK(machinePassword))
          {
            configuration["Service"]["sMachinePassword"] = machinePassword;
          }
          else
          {
            validateConfigPassword(ref configuration, ref machinePassword, ref promptShown);
          }
        }

        if (!autorunArgFound)
        {
          autoRun = bool.Parse(configuration["Service"]["bAutoRun"]);
        }
        else
        {
          autoRun = !bool.Parse(configuration["Service"]["bAutoRun"]);
          configuration["Service"]["bAutoRun"] = autoRun.ToString();
        }
      }


      // TODO: Get rid of these
      machine = machineName;
      pass = <PASSWORD>Password;

      // Enable/disable autorun
      if (runningWindows)
      {
        if (autoRun)
          SwitchAutorun(Application.ProductName, Path.Combine(desiredAppDirectory.FullName, Path.GetFileName(Application.ExecutablePath)));
        else
          SwitchAutorun(Application.ProductName);
      }

      #region Update INI file
      if (firstRun || !machineConfigured || autorunArgFound || passArgFound || promptShown)
      {
        if (!machineConfigured)
        {
          configureMachine();
          machineConfigured = true;
          configuration["Service"]["bMachineConfigured"] = machineConfigured.ToString();
        }

        config_parser.WriteFile(mainConfigFile.FullName, configuration, Encoding.UTF8);
      }
      #endregion

      #region 1 - Time Thread
      var timeThread = new Thread(sendMachineTime);
      timeThread.IsBackground = true;
      timeThread.Start();
      #endregion
      Thread.Sleep(1000);

      #region 2 - Connection Thread
      connectionThread = new Thread(handleConnection);
      connectionThread.IsBackground = true;
      connectionThread.Start();
      #endregion

      #region 3 - Command Thread
      commandThread = new Thread(awaitCommands);
      commandThread.IsBackground = true;
      // Starts in a different place
      #endregion

      #region 4 - Chat Thread
      chatThread = new Thread(serveMessages);
      chatThread.IsBackground = true;
      // Starts in a different place
      #endregion

      timeThread.Join();
    }


    private static void sendMachineTime()
    {
      reportThreadStart(resources.TimeStart);

      while (true)
      {
        var now = DateTime.Now;

        if (resources.WebErrorMessage != Set($"time.{machine}" /* PUN NOT INTENDED */,
          $"{now.ToShortDateString()} {now.ToLongTimeString()}"))
        {
          if (!internetAlive)
          {
            resurrectDeadThreads();
          }<commit_msg>Refactored Main function, removed regions<commit_after>using IniParser;
using IniParser.Model;
using Svetomech.Utilities;
using System;
using System.Configuration;
using System.Globalization;
using System.IO;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading;
using static Svetomech.Utilities.NativeMethods;
using static Svetomech.Utilities.SimpleApp;
using static Svetomech.Utilities.SimplePlatform;

namespace SimpleMaid
{
  internal static partial class Program
  {
    private static bool Hidden { get; set; } = false;
    private static string State { set { Console.Title = $"{Application.ProductName}: {value}"; } }

    private static DirectoryInfo desiredAppDirectory;
    private static FileInfo mainConfigFile;
    private static IntPtr mainWindowHandle;
    private static Mutex singleInstance;

    private static Thread connectionThread;
    private static Thread commandThread;
    private static Thread chatThread;
    private static volatile bool busyConnectionWise = true;
    private static volatile bool busyCommandWise = false;
    private static volatile bool busyChatWise = false;

    private static readonly bool runningWindows = (RunningPlatform() == Platform.Windows);
    private static volatile bool internetAlive = true;

    // TODO: Get rid of these two
    private static volatile string machine;
    private static volatile string pass;

    internal static frmChatWindow ChatboxWindow = null;
    internal static volatile bool ChatboxExit = false;
    internal static volatile string SupportChatMessage;
    internal static volatile string UserChatMessage;
    internal static volatile string ChatCommand;


    private static string Set(string tag, string value)
    {
      tag = $"{Application.ProductName}_{tag}";

      var encoding = new UTF8Encoding();
      byte[] requestBody = encoding.GetBytes($"tag={tag}&value={value}&fmt=html");

      var request = (HttpWebRequest)WebRequest.Create($"{Variables.ServerAddress}/storeavalue");
      request.Method = "POST";
      request.Credentials = Variables.AccountCredentials;
      request.ContentType = "application/x-www-form-urlencoded";
      request.ContentLength = requestBody.Length;
      request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";

      try
      {
        using (var requestStream = request.GetRequestStream())
        {
          requestStream.Write(requestBody, 0, requestBody.Length);
        }
      }
      catch (WebException)
      {
        reportWebError();
        internetAlive = false;
        return resources.WebErrorMessage;
      }
      try { using (var response = request.GetResponse()) ;}
      catch (WebException)
      {
        reportWebError();
        internetAlive = false;
        return resources.WebErrorMessage;
      }

      resetConsoleColor();
      Console.ForegroundColor = ConsoleColor.Gray;
      Console.WriteLine($"SET  {tag}  {value}\n");

      return String.Empty;
    }

    private static string Get(string tag)
    {
      tag = $"{Application.ProductName}_{tag}";

      string value;

      var encoding = new UTF8Encoding();
      byte[] requestBody = encoding.GetBytes($"tag={tag}&fmt=html");

      var request = (HttpWebRequest)WebRequest.Create($"{Variables.ServerAddress}/getvalue");
      request.Method = "POST";
      request.Credentials = Variables.AccountCredentials;
      request.ContentType = "application/x-www-form-urlencoded";
      request.ContentLength = requestBody.Length;
      request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";

      try
      {
        using (var requestStream = request.GetRequestStream())
        {
          requestStream.Write(requestBody, 0, requestBody.Length);
        }
      }
      catch (WebException)
      {
        reportWebError();
        internetAlive = false;
        return resources.WebErrorMessage;
      }

      // TODO: Refactor
      try
      {
        using (var response = request.GetResponse())
        using (var responseStream = response.GetResponseStream())
        using (var sr = new StreamReader(responseStream))
        {
          value = sr.ReadToEnd();
          value = value.Substring(value.IndexOf(tag) + tag.Length + 4);
          value = value.Remove(value.IndexOf("\""));
        }
      }
      catch (WebException)
      {
        reportWebError();
        internetAlive = false;
        return resources.WebErrorMessage;
      }

      // TODO: Rewrite - problems with decoding
      value = decodeEncodedNonAsciiCharacters(value);
      value = value.Replace(@"\/", @"/");
      value = value.Replace(@"\\", @"\");
      value = WebUtility.HtmlDecode(value);

      resetConsoleColor();
      Console.ForegroundColor = ConsoleColor.DarkGreen;
      Console.WriteLine($"GET  {tag}  {value}\n");

      return value;
    }

    private static void SetUntilSet(string tag, string value)
    {
      while (resources.WebErrorMessage == Set(tag, value))
      {
        Thread.Sleep(Variables.GeneralDelay);
      }
    }

    private static string GetUntilGet(string tag)
    {
      string value;

      while (resources.WebErrorMessage == (value = Get(tag)))
      {
        Thread.Sleep(Variables.GeneralDelay);
      }

      return value;
    }


    [STAThread]
    private static void Main(string[] args)
    {
      System.Windows.Forms.Application.EnableVisualStyles();
      Console.Clear();


      if (IsElevated())
      {
        reportGeneralError(resources.AdminErrorMessage);
        exit();
      }


      desiredAppDirectory = new DirectoryInfo(Path.Combine(Environment.GetFolderPath(
        Environment.SpecialFolder.LocalApplicationData), Application.CompanyName, Application.ProductName));
      mainConfigFile = new FileInfo(Path.Combine(desiredAppDirectory.FullName,
        (Variables.ConfigName != Variables.KeywordDefault) ? Variables.ConfigName : $"{Application.ProductName}.ini"));


      bool rogueArgFound = false;
      bool autorunArgFound = false;
      bool passArgFound = false; // <-- only need it for clarity
      bool langArgFound = false; // <-- only need it for clarity

      string passArg = Variables.DefaultPassword;
      string langArg = CultureInfo.InstalledUICulture.Name;

      if (args.Length >= 1)
      {
        int adjustedLength = args.Length;

        for (int i = 0; i < args.Length; ++i)
        {
          if (!rogueArgFound && (rogueArgFound = (args[i] == Variables.RogueArgument)))
            adjustedLength--;

          if (!autorunArgFound && (autorunArgFound = (args[i] == Variables.AutorunArgument)))
            adjustedLength--;

          if (adjustedLength >= 2)
          {
            if (!passArgFound && (args[i] == Variables.PasswordArgument))
            {
              passArg = (passArgFound = (i + 1 < adjustedLength)) ? args[i + 1] : passArg;

              if (passArgFound)
                adjustedLength -= 2;
              else
                adjustedLength--;
            }
            else if (!langArgFound && (args[i] == Variables.LanguageArgument))
            {
              langArg = (langArgFound = (i + 1 < adjustedLength)) ? args[i + 1] : langArg;

              if (langArgFound)
                adjustedLength -= 2;
              else
                adjustedLength--;
            }
          }
        }
      }


      mainWindowHandle = GetConsoleWindow();
      bool inDesiredDir = desiredAppDirectory.IsEqualTo(Application.StartupPath);
      if (inDesiredDir || rogueArgFound)
      {
        ShowWindow(mainWindowHandle, SW_HIDE);
        Program.Hidden = true;
      }


      CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.GetCultureInfo(langArg);


      // TODO: Move so it happens AFTER startup directory management
      singleInstance = new Mutex(false, "Local\\" + Application.AssemblyGuid);
      if (!singleInstance.WaitOne(0, false))
      {
        reportPastSelf();
        exit();
      }


      if (!inDesiredDir)
      {
        var svtFolder = new DirectoryInfo(ConfigurationManager.AppSettings["SvtFolderName"]);
        string[] filePaths = { Application.ExecutablePath, ConfigurationManager.OpenExeConfiguration(ConfigurationUserLevel.None).FilePath,
          Assembly.GetAssembly(typeof(FileIniDataParser)).Location };
        try
        {
          var desiredAppSubdirectory = new DirectoryInfo(Path.Combine(desiredAppDirectory.FullName, svtFolder.Name));

          svtFolder.CopyTo(desiredAppSubdirectory, false);

          foreach (var filePath in filePaths)
          {
            File.Copy(filePath, Path.Combine(desiredAppDirectory.FullName, Path.GetFileName(filePath)), true);
          }
        }
        catch //unauthorized, io, notsupported
        {
          // 1. stop other instance using guid technique
          //   if there is no other instance running, (?recursive or ?continue)
          // 2. try to copy again
          //   if unlucky, (?recursive or ?continue)
          // 3. start that instance (?with a delay through .bat file)
          // 4. exit
        }
      }


      var config_parser = new FileIniDataParser();
      IniData configuration;

      bool   machineConfigured = false;
      string machineName       = createMachine();
      string machinePassword   = <PASSWORD>;
      bool   autoRun           = autorunArgFound;

      bool firstRun;
      bool promptShown = false;
      if (firstRun = !mainConfigFile.Exists)
      {
        configuration = new IniData();
        configuration.Sections.AddSection("Service");


        configuration["Service"].AddKey("bMachineConfigured", machineConfigured.ToString());
        configuration["Service"].AddKey("sMachineName", machineName);

        validateMemoryPassword(ref configuration, ref machinePassword, ref promptShown);

        configuration["Service"].AddKey("bAutoRun", autoRun.ToString());
      }
      else
      {
        configuration = config_parser.ReadFile(mainConfigFile.FullName, Encoding.UTF8);


        machineConfigured = bool.Parse(configuration["Service"]["bMachineConfigured"]);

        if (isNameOK(configuration["Service"]["sMachineName"]))
        {
          machineName = configuration["Service"]["sMachineName"];
        }
        else
        {
          configuration["Service"]["sMachineName"] = machineName;
          machineConfigured = false;
        }

        if (!passArgFound)
        {
          validateConfigPassword(ref configuration, ref machinePassword, ref promptShown);
        }
        else
        {
          if (isPasswordOK(machinePassword))
          {
            configuration["Service"]["sMachinePassword"] = <PASSWORD>Password;
          }
          else
          {
            validateConfigPassword(ref configuration, ref machinePassword, ref promptShown);
          }
        }

        if (!autorunArgFound)
        {
          autoRun = bool.Parse(configuration["Service"]["bAutoRun"]);
        }
        else
        {
          autoRun = !bool.Parse(configuration["Service"]["bAutoRun"]);
          configuration["Service"]["bAutoRun"] = autoRun.ToString();
        }
      }

      if (firstRun || !machineConfigured || autorunArgFound || passArgFound || promptShown)
      {
        if (!machineConfigured)
        {
          configureMachine();
          machineConfigured = true;
          configuration["Service"]["bMachineConfigured"] = machineConfigured.ToString();
        }

        config_parser.WriteFile(mainConfigFile.FullName, configuration, Encoding.UTF8);
      }

      // TODO: Get rid of these
      machine = machineName;
      pass = <PASSWORD>;


      if (runningWindows)
      {
        if (autoRun)
        {
          SwitchAutorun(Application.ProductName, Path.Combine(desiredAppDirectory.FullName,
            Path.GetFileName(Application.ExecutablePath)));
        }
        else
        {
          SwitchAutorun(Application.ProductName);
        }
      }


      var timeThread = new Thread(sendMachineTime);
      timeThread.IsBackground = true;
      timeThread.Start();
      Thread.Sleep(1000);

      connectionThread = new Thread(handleConnection);
      connectionThread.IsBackground = true;
      connectionThread.Start();

      commandThread = new Thread(awaitCommands);
      commandThread.IsBackground = true;
      // Starts in a different place

      chatThread = new Thread(serveMessages);
      chatThread.IsBackground = true;
      // Starts in a different place


      timeThread.Join();
    }


    private static void sendMachineTime()
    {
      reportThreadStart(resources.TimeStart);

      while (true)
      {
        var now = DateTime.Now;

        if (resources.WebErrorMessage != Set($"time.{machine}" /* PUN NOT INTENDED */,
          $"{now.ToShortDateString()} {now.ToLongTimeString()}"))
        {
          if (!internetAlive)
          {
            resurrectDeadThreads();
          }
{'fraction_non_alphanumeric': 0.1028582244936589, 'fraction_numerical': 0.005224304372515616, 'mean_word_length': 2.4391355292279653, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '75103', 'n_tokens_mistral': 7118, 'n_tokens_neox': 6882, 'n_words': 1568}