<commit_before>#define WIN32_LEAN_AND_MEAN
#define STRICT
#define _ATL_NO_AUTOMATIC_NAMESPACE
#define WIN32_NO_STATUS
#include <windows.h>
#undef WIN32_NO_STATUS
#include <ntstatus.h>
#include <winternl.h>
#include <pathcch.h>
#include <atlalloc.h>
#include <atlbase.h>
#include <atlchecked.h>
#include <atlcore.h>
#include <array>
#include <memory>
#include <string>
#include <cstdint>
#include "lxsstat.hpp"
#include "fileopen.hpp"
#pragma comment(lib, "pathcch")
#pragma comment(lib, "ntdll")

#pragma region
EXTERN_C NTSYSAPI NTSTATUS NTAPI ZwQueryEaFile(
	_In_     HANDLE           FileHandle,
	_Out_    PIO_STATUS_BLOCK IoStatusBlock,
	_Out_    PVOID            Buffer,
	_In_     ULONG            Length,
	_In_     BOOLEAN          ReturnSingleEntry,
	_In_opt_ PVOID            EaList,
	_In_     ULONG            EaListLength,
	_In_opt_ PULONG           EaIndex,
	_In_     BOOLEAN          RestartScan
);
struct FILE_FULL_EA_INFORMATION
{
	ULONG  NextEntryOffset;
	UCHAR  Flags;
	UCHAR  EaNameLength;
	USHORT EaValueLength;
	CHAR   EaName[ANYSIZE_ARRAY];
};
struct FILE_GET_EA_INFORMATION
{
	ULONG NextEntryOffset;
	UCHAR EaNameLength;
	CHAR  EaName[ANYSIZE_ARRAY];
};
constexpr size_t MAXIMUM_LENGTH_OF_EA_NAME = (std::numeric_limits<decltype(FILE_FULL_EA_INFORMATION::EaNameLength)>::max)();
constexpr size_t MAXIMUM_LENGTH_OF_EA_VALUE = (std::numeric_limits<decltype(FILE_FULL_EA_INFORMATION::EaValueLength)>::max)();
#pragma endregion

namespace Lxss
{
	const auto lxss_root = []()->std::wstring
	{
		WCHAR temp[MAX_PATH];
		ATLENSURE(ExpandEnvironmentStringsW(LR"(%LOCALAPPDATA%\lxss)", temp, ARRAYSIZE(temp)));
		return temp;
	}();
	std::wstring realpath(std::wstring path)
	{
		_RPT1(_CRT_WARN, "realpath(<%ls)\n", path.c_str());
		const WCHAR prefix[] = LR"(\\?\)";
		if (path[0] == L'/')
		{
			const WCHAR fixups[] = LR"(#<>:"\|?*)"
				L"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F"
				L"\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F";
			for (size_t pos = path.find_first_of(fixups); pos != std::string::npos; pos = path.find_first_of(fixups, pos))
			{
				WCHAR buf[5];
				ATLENSURE(swprintf_s(buf, L"%04X", path[pos]) == _countof(buf) - 1);
				path[pos] = L'#';
				path.insert(++pos, buf);
				pos += _countof(buf) - 1;
			}
			bool need_reloc = false;
			const PCWSTR reloc_directory[] = {
				L"/root",
				L"/home",
			};
			for (int i = 0; i < _countof(reloc_directory); ++i)
			{
				const size_t reloc_length = wcslen(reloc_directory[i]);
				if (
					wcsncmp(path.c_str(), reloc_directory[i], reloc_length) == 0
					&& (path[reloc_length] == L'\0' || path[reloc_length] == L'/'))
				{
					need_reloc = true;
					break;
				}
			}
			path.insert(0, !need_reloc ? lxss_root + L"\\rootfs" : lxss_root);
		}
		else if (PathIsRelativeW(path.c_str()) || (path[0] == '\\' && wcsncmp(path.c_str(), prefix, wcslen(prefix)) != 0))
		{
			auto temp = std::make_unique<WCHAR[]>(PATHCCH_MAX_CCH);<commit_msg>Bugfix filename handling<commit_after>#define WIN32_LEAN_AND_MEAN
#define STRICT
#define _ATL_NO_AUTOMATIC_NAMESPACE
#define WIN32_NO_STATUS
#include <windows.h>
#undef WIN32_NO_STATUS
#include <ntstatus.h>
#include <winternl.h>
#include <pathcch.h>
#include <atlalloc.h>
#include <atlbase.h>
#include <atlchecked.h>
#include <atlcore.h>
#include <array>
#include <memory>
#include <string>
#include <cstdint>
#include "lxsstat.hpp"
#include "fileopen.hpp"
#pragma comment(lib, "pathcch")
#pragma comment(lib, "ntdll")

#pragma region
EXTERN_C NTSYSAPI NTSTATUS NTAPI ZwQueryEaFile(
	_In_     HANDLE           FileHandle,
	_Out_    PIO_STATUS_BLOCK IoStatusBlock,
	_Out_    PVOID            Buffer,
	_In_     ULONG            Length,
	_In_     BOOLEAN          ReturnSingleEntry,
	_In_opt_ PVOID            EaList,
	_In_     ULONG            EaListLength,
	_In_opt_ PULONG           EaIndex,
	_In_     BOOLEAN          RestartScan
);
struct FILE_FULL_EA_INFORMATION
{
	ULONG  NextEntryOffset;
	UCHAR  Flags;
	UCHAR  EaNameLength;
	USHORT EaValueLength;
	CHAR   EaName[ANYSIZE_ARRAY];
};
struct FILE_GET_EA_INFORMATION
{
	ULONG NextEntryOffset;
	UCHAR EaNameLength;
	CHAR  EaName[ANYSIZE_ARRAY];
};
constexpr size_t MAXIMUM_LENGTH_OF_EA_NAME = (std::numeric_limits<decltype(FILE_FULL_EA_INFORMATION::EaNameLength)>::max)();
constexpr size_t MAXIMUM_LENGTH_OF_EA_VALUE = (std::numeric_limits<decltype(FILE_FULL_EA_INFORMATION::EaValueLength)>::max)();
#pragma endregion

namespace Lxss
{
	const auto lxss_root = []()->std::wstring
	{
		WCHAR temp[MAX_PATH];
		ATLENSURE(ExpandEnvironmentStringsW(LR"(%LOCALAPPDATA%\lxss)", temp, ARRAYSIZE(temp)));
		return temp;
	}();
	std::wstring realpath(std::wstring path)
	{
		_RPT1(_CRT_WARN, "realpath(<%ls)\n", path.c_str());
		const WCHAR prefix[] = LR"(\\?\)";
		if (path[0] == L'/')
		{
			const WCHAR fixups[] = LR"(#<>:"\|?*)"
				L"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10"
				L"\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F";
			for (size_t pos = path.find_first_of(fixups); pos != std::string::npos; pos = path.find_first_of(fixups, pos))
			{
				WCHAR buf[5];
				ATLENSURE(swprintf_s(buf, L"%04X", path[pos]) == _countof(buf) - 1);
				path[pos] = L'#';
				path.insert(++pos, buf);
				pos += _countof(buf) - 1;
			}
			bool need_reloc = false;
			const PCWSTR reloc_directory[] = {
				L"/root",
				L"/home",
			};
			for (int i = 0; i < _countof(reloc_directory); ++i)
			{
				const size_t reloc_length = wcslen(reloc_directory[i]);
				if (
					wcsncmp(path.c_str(), reloc_directory[i], reloc_length) == 0
					&& (path[reloc_length] == L'\0' || path[reloc_length] == L'/'))
				{
					need_reloc = true;
					break;
				}
			}
			path.insert(0, !need_reloc ? lxss_root + L"\\rootfs" : lxss_root);
		}
		else if (PathIsRelativeW(path.c_str()) || (path[0] == '\\' && wcsncmp(path.c_str(), prefix, wcslen(prefix)) != 0))
		{
			auto temp = std::make_unique<WCHAR[]>(PATHCCH_MAX_CCH);
{'fraction_non_alphanumeric': 0.15328958438499074, 'fraction_numerical': 0.02288406528689214, 'mean_word_length': 3.9698996655518393, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '593230', 'n_tokens_mistral': 2755, 'n_tokens_neox': 2484, 'n_words': 421}