<commit_before>#include "Platform/Platform.h"
#if FL_ENABLED(FL_PLATFORM_WINDOWS)

#include "FileSystem/FileSystem.h"

namespace Flourish::FileSystem
{
	void ReportFileError(const char* path, const char* functionName, int errorCode)
	{
		char* messageBuffer = nullptr;
		FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
                             nullptr, errorCode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), reinterpret_cast<char*>(&messageBuffer), 0, nullptr);

		fprintf(stderr, "%s: %s error: %s\n", path, functionName, messageBuffer);
		LocalFree(messageBuffer);
	}

    bool CreateDirectoryTree(const char* path)
    {
		char dirPath[MAX_PATH];

	    strcpy_s(dirPath, path);
		int pathLen = strlen(dirPath);

		for(int i = 0; i < pathLen; ++i)
		{
			if(dirPath[i] == '/')
			{
				dirPath[i] = '\\';
			}
		}

		if(dirPath[pathLen- 1] != '\\')
		{
			strcat_s(dirPath,"\\");
		}

		char folder[MAX_PATH];
		memset(folder, 0, MAX_PATH * sizeof(char));


		const char* end = strchr(dirPath, '\\');

		while(end != nullptr)
		{
		    strncpy_s(folder, dirPath, end - dirPath + 1);
		    if(!CreateDirectoryA(folder, nullptr))
		    {
				DWORD error = GetLastError();
				if(error != ERROR_ALREADY_EXISTS)
				{
					ReportFileError(dirPath, "CreateDirectoryTree", error);
					return false;
				}
		    }
		    end = strchr(++end, '\\');
		}

		return true;
    }<commit_msg>Added a 1ms sleep after creating a directory in FileSystem_WIN::CreateDirectoryTree. It's gross but if we don't do that we sometimes get to the point of creating the next directory down before that one has been created... For some reason<commit_after>#include "Platform/Platform.h"
#if FL_ENABLED(FL_PLATFORM_WINDOWS)

#include "FileSystem/FileSystem.h"

namespace Flourish::FileSystem
{
	void ReportFileError(const char* path, const char* functionName, int errorCode)
	{
		char* messageBuffer = nullptr;
		FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
                             nullptr, errorCode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), reinterpret_cast<char*>(&messageBuffer), 0, nullptr);

		fprintf(stderr, "%s: %s error: %s\n", path, functionName, messageBuffer);
		LocalFree(messageBuffer);
	}

    bool CreateDirectoryTree(const char* path)
    {
		char dirPath[MAX_PATH];

	    strcpy_s(dirPath, path);
		int pathLen = strlen(dirPath);

		for(int i = 0; i < pathLen; ++i)
		{
			if(dirPath[i] == '/')
			{
				dirPath[i] = '\\';
			}
		}

		if(dirPath[pathLen- 1] != '\\')
		{
			strcat_s(dirPath,"\\");
		}

		char folder[MAX_PATH];
		memset(folder, 0, MAX_PATH * sizeof(char));


		const char* end = strchr(dirPath, '\\');

		while(end != nullptr)
		{
		    strncpy_s(folder, dirPath, end - dirPath + 1);
		    if(!CreateDirectoryA(folder, nullptr))
		    {
				DWORD error = GetLastError();
				if(error != ERROR_ALREADY_EXISTS)
				{
					ReportFileError(dirPath, "CreateDirectoryTree", error);
					return false;
				}
		    }
			Sleep(1);
		    end = strchr(++end, '\\');
		}

		return true;
    }
{'fraction_non_alphanumeric': 0.13544668587896252, 'fraction_numerical': 0.0038424591738712775, 'mean_word_length': 3.5341074020319305, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2881540', 'n_tokens_mistral': 1227, 'n_tokens_neox': 1075, 'n_words': 227}