<commit_before>#ifdef _WIN32

#include "platform.h"
#include "imgui.h"
#include <Windows.h>
#include <assert.h>
#include <iostream>
#include <stdint.h>
#include <winnls.h>

wchar_t *utf8_to_wide(const char *s) {
	size_t len = utf8len(s);
	wchar_t *buf = (wchar_t *)malloc(sizeof(wchar_t) * (1 + len));
	mbstowcs(buf, s, len);
	buf[len] = 0;
	return buf;
}

char *wide_to_utf8(const wchar_t *s) {
	size_t len = 0;
	wchar_t c;
	for (const wchar_t *ss = s; 0 != (c = *ss); ++ss) {
		if (c < 0x80) {
			len += 1;
		} else if (c < 0x800) {
			len += 2;
		} else if (c >= 0xd800 && c < 0xdc00) {
			// surrogate pair
			len += 4;
			++ss;
		} else if (c >= 0xdc00 && c < 0xe000) {
			// pair end -- invalid sequence
			return nullptr;
		} else {
			len += 3;
		}
	}
	char *buf = (char *)malloc(1 + len);
	wcstombs(buf, s, len);
	buf[len] = 0;
	return buf;
}

char *file_as_buffer(size_t *buffer_size, const char *utf8_filename) {
	wchar_t *wide_filename = utf8_to_wide(utf8_filename);
	HANDLE file = CreateFile(wide_filename, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
	                         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
	free(wide_filename);
	if (file == INVALID_HANDLE_VALUE) {
		*buffer_size = 0;
		return nullptr;
	}

	LARGE_INTEGER filesize;
	GetFileSizeEx(file, &filesize);
	uint32_t sz = (uint32_t)filesize.QuadPart;
	assert(filesize.QuadPart == sz);
	*buffer_size = sz;

	char *buf = (char *)malloc(sz);
	uint32_t numRead = 0;
	ReadFile(file, buf, sz, (LPDWORD)&numRead, NULL);
	assert(numRead == sz);

	return buf;
}

char *show_file_picker() {
	OPENFILENAME ofn;
	wchar_t filename[1024];
	filename[0] = 0;
	memset(&ofn, 0, sizeof(ofn));
	ofn.lStructSize = sizeof(ofn);
	ofn.hwndOwner = (HWND)ImGui::GetIO().ImeWindowHandle;
	ofn.lpstrFilter = L"BRD Files\0*.brd\0All Files\0*.*\0\0";
	ofn.lpstrFile = filename;
	ofn.nMaxFile = 1024;
	if (GetOpenFileName(&ofn)) {
		return wide_to_utf8(filename);
	}
	return nullptr;
}

// Not caring about XP. If this is a concern, find a replacement for CompareStringEx()
const std::vector<char> load_font(const std::string &name) {
	std::vector<char> data;
	HFONT fontHandle;

	wchar_t *wname = utf8_to_wide(name.c_str());

	fontHandle = CreateFont(0, 0, 0, 0, 0, 0, 0, 0, 0, OUT_TT_ONLY_PRECIS, 0, 0, 0, wname);
	if (!fontHandle) {
		std::cerr << "CreateFont failed" << std::endl;
		free(wname);
		return data;
	}<commit_msg>Revert "Win32: search for TrueType fonts only"<commit_after>#ifdef _WIN32

#include "platform.h"
#include "imgui.h"
#include <Windows.h>
#include <assert.h>
#include <iostream>
#include <stdint.h>
#include <winnls.h>

wchar_t *utf8_to_wide(const char *s) {
	size_t len = utf8len(s);
	wchar_t *buf = (wchar_t *)malloc(sizeof(wchar_t) * (1 + len));
	mbstowcs(buf, s, len);
	buf[len] = 0;
	return buf;
}

char *wide_to_utf8(const wchar_t *s) {
	size_t len = 0;
	wchar_t c;
	for (const wchar_t *ss = s; 0 != (c = *ss); ++ss) {
		if (c < 0x80) {
			len += 1;
		} else if (c < 0x800) {
			len += 2;
		} else if (c >= 0xd800 && c < 0xdc00) {
			// surrogate pair
			len += 4;
			++ss;
		} else if (c >= 0xdc00 && c < 0xe000) {
			// pair end -- invalid sequence
			return nullptr;
		} else {
			len += 3;
		}
	}
	char *buf = (char *)malloc(1 + len);
	wcstombs(buf, s, len);
	buf[len] = 0;
	return buf;
}

char *file_as_buffer(size_t *buffer_size, const char *utf8_filename) {
	wchar_t *wide_filename = utf8_to_wide(utf8_filename);
	HANDLE file = CreateFile(wide_filename, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
	                         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
	free(wide_filename);
	if (file == INVALID_HANDLE_VALUE) {
		*buffer_size = 0;
		return nullptr;
	}

	LARGE_INTEGER filesize;
	GetFileSizeEx(file, &filesize);
	uint32_t sz = (uint32_t)filesize.QuadPart;
	assert(filesize.QuadPart == sz);
	*buffer_size = sz;

	char *buf = (char *)malloc(sz);
	uint32_t numRead = 0;
	ReadFile(file, buf, sz, (LPDWORD)&numRead, NULL);
	assert(numRead == sz);

	return buf;
}

char *show_file_picker() {
	OPENFILENAME ofn;
	wchar_t filename[1024];
	filename[0] = 0;
	memset(&ofn, 0, sizeof(ofn));
	ofn.lStructSize = sizeof(ofn);
	ofn.hwndOwner = (HWND)ImGui::GetIO().ImeWindowHandle;
	ofn.lpstrFilter = L"BRD Files\0*.brd\0All Files\0*.*\0\0";
	ofn.lpstrFile = filename;
	ofn.nMaxFile = 1024;
	if (GetOpenFileName(&ofn)) {
		return wide_to_utf8(filename);
	}
	return nullptr;
}

// Not caring about XP. If this is a concern, find a replacement for CompareStringEx()
const std::vector<char> load_font(const std::string &name) {
	std::vector<char> data;
	HFONT fontHandle;

	wchar_t *wname = utf8_to_wide(name.c_str());

	fontHandle = CreateFont(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, name.empty() ? NULL : wname);
	if (!fontHandle) {
		std::cerr << "CreateFont failed" << std::endl;
		free(wname);
		return data;
	}
{'fraction_non_alphanumeric': 0.15852649006622516, 'fraction_numerical': 0.032491721854304635, 'mean_word_length': 4.044885177453027, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6923041', 'n_tokens_mistral': 2210, 'n_tokens_neox': 2029, 'n_words': 480}