<commit_before>#include "../../includes/win32/FSWatcher.h"

#pragma managed
static void beginDispatchThread(Object ^fsWatcher) {
  ((FSWatcher ^)fsWatcher)->dispatchSetup();
}

[PermissionSet(SecurityAction::Demand, Name="FullTrust")]
FSWatcher::FSWatcher(EventQueue &queue, System::String ^path):
  mDispatching(false),
  mQueue(queue) {
  mDispatchQueue = gcnew ConcurrentQueue< Tuple<Int32, Object ^> ^>();

  mDispatchThread = gcnew Thread(gcnew ParameterizedThreadStart(&beginDispatchThread));
  mDispatchThread->IsBackground = true;
  mDispatchThread->Start(this);

  mWatcher = gcnew FileSystemWatcher();
  mWatcher->Path = path;
  mWatcher->InternalBufferSize = 64 * 1024;
  mWatcher->IncludeSubdirectories = true;
  mWatcher->NotifyFilter = static_cast<NotifyFilters>(
    NotifyFilters::FileName |
    NotifyFilters::DirectoryName |
    NotifyFilters::Attributes |
    NotifyFilters::LastWrite |
    NotifyFilters::Security |
    NotifyFilters::Size
  );

  mWatcher->Changed += gcnew FileSystemEventHandler(this, &FSWatcher::onChangedDispatch);
  mWatcher->Created += gcnew FileSystemEventHandler(this, &FSWatcher::onCreatedDispatch);
  mWatcher->Deleted += gcnew FileSystemEventHandler(this, &FSWatcher::onDeletedDispatch);
  mWatcher->Error += gcnew ErrorEventHandler(this, &FSWatcher::onErrorDispatch);
  mWatcher->Renamed += gcnew RenamedEventHandler(this, &FSWatcher::onRenamedDispatch);

  mWatcher->EnableRaisingEvents = true;
}

FSWatcher::~FSWatcher() {
  delete mWatcher;
  mDispatching = false;
  mDispatchThread->Join();
}

void FSWatcher::dispatchSetup() {
  mDispatching = true;
  while (mDispatching) {
    Int32 count = mDispatchQueue->Count;
    if (count == 0) {
      continue;
    }

    for (Int32 i = 0; i < count; ++i) {
      if (!mDispatching) {
        break;
      }
      Tuple<Int32, Object^> ^toDispatch;
      while (!mDispatchQueue->TryDequeue(toDispatch)) {}
      if (toDispatch->Item1 == RENAMED) {
        onRenamed(safe_cast<RenamedEventArgs ^>(toDispatch->Item2));
      } else {
        eventHandlerHelper(static_cast<EventType>(toDispatch->Item1), safe_cast<FileSystemEventArgs ^>(toDispatch->Item2));
      }
    }
  }
}

System::String^ getDirectoryName(System::String^ path)
{
  wchar_t delim = '\\';
  array<System::String^>^ tokens = path->Split(delim);

  if (path[path->Length - 1] == delim)
  {
    if (
      tokens->Length == 2 &&
      System::String::IsNullOrEmpty(tokens[1]) ||
      tokens->Length < 2
    ) {
      return gcnew System::String("");
    }
    else
    {
      return path->Substring(0, path->Length - 1);
    }
  }
  else
  {
    return path->Substring(0, path->LastIndexOf(delim));<commit_msg>Add sleeps to windows dispatch loop<commit_after>#include "../../includes/win32/FSWatcher.h"

#pragma managed
static void beginDispatchThread(Object ^fsWatcher) {
  ((FSWatcher ^)fsWatcher)->dispatchSetup();
}

[PermissionSet(SecurityAction::Demand, Name="FullTrust")]
FSWatcher::FSWatcher(EventQueue &queue, System::String ^path):
  mDispatching(false),
  mQueue(queue) {
  mDispatchQueue = gcnew ConcurrentQueue< Tuple<Int32, Object ^> ^>();

  mDispatchThread = gcnew Thread(gcnew ParameterizedThreadStart(&beginDispatchThread));
  mDispatchThread->IsBackground = true;
  mDispatchThread->Start(this);

  mWatcher = gcnew FileSystemWatcher();
  mWatcher->Path = path;
  mWatcher->InternalBufferSize = 64 * 1024;
  mWatcher->IncludeSubdirectories = true;
  mWatcher->NotifyFilter = static_cast<NotifyFilters>(
    NotifyFilters::FileName |
    NotifyFilters::DirectoryName |
    NotifyFilters::Attributes |
    NotifyFilters::LastWrite |
    NotifyFilters::Security |
    NotifyFilters::Size
  );

  mWatcher->Changed += gcnew FileSystemEventHandler(this, &FSWatcher::onChangedDispatch);
  mWatcher->Created += gcnew FileSystemEventHandler(this, &FSWatcher::onCreatedDispatch);
  mWatcher->Deleted += gcnew FileSystemEventHandler(this, &FSWatcher::onDeletedDispatch);
  mWatcher->Error += gcnew ErrorEventHandler(this, &FSWatcher::onErrorDispatch);
  mWatcher->Renamed += gcnew RenamedEventHandler(this, &FSWatcher::onRenamedDispatch);

  mWatcher->EnableRaisingEvents = true;
}

FSWatcher::~FSWatcher() {
  delete mWatcher;
  mDispatching = false;
  mDispatchThread->Join();
}

void FSWatcher::dispatchSetup() {
  mDispatching = true;
  while (mDispatching) {
    Int32 count = mDispatchQueue->Count;
    if (count == 0) {
      Sleep(50);
      continue;
    }

    for (Int32 i = 0; i < count; ++i) {
      if (!mDispatching) {
        return;
      }
      Tuple<Int32, Object^> ^toDispatch;
      while (!mDispatchQueue->TryDequeue(toDispatch)) {
        if (!mDispatching) {
          return;
        }
        Sleep(1);
      }

      if (toDispatch->Item1 == RENAMED) {
        onRenamed(safe_cast<RenamedEventArgs ^>(toDispatch->Item2));
      } else {
        eventHandlerHelper(static_cast<EventType>(toDispatch->Item1), safe_cast<FileSystemEventArgs ^>(toDispatch->Item2));
      }
    }
  }
}

System::String^ getDirectoryName(System::String^ path)
{
  wchar_t delim = '\\';
  array<System::String^>^ tokens = path->Split(delim);

  if (path[path->Length - 1] == delim)
  {
    if (
      tokens->Length == 2 &&
      System::String::IsNullOrEmpty(tokens[1]) ||
      tokens->Length < 2
    ) {
      return gcnew System::String("");
    }
    else
    {
      return path->Substring(0, path->Length - 1);
    }
  }
  else
  {
    return path->Substring(0, path->LastIndexOf(delim));
{'fraction_non_alphanumeric': 0.1417869541384981, 'fraction_numerical': 0.011145623972227297, 'mean_word_length': 4.1302717900656045, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4477084', 'n_tokens_mistral': 1822, 'n_tokens_neox': 1780, 'n_words': 295}