<commit_before>#pragma once

#ifndef _WIN32
#include <cstdlib>
#include <sys/stat.h>
#include <sys/file.h>
#include <sys/ioctl.h>
#include <dirent.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/statvfs.h>
#if __linux__ || __APPLE__
extern "C" int rep_closefrom(int lower);
#define closefrom rep_closefrom
#endif
#else
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN 1
#endif
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <Windows.h>
#include <cwchar>
#include <cwctype>
#include <Shlwapi.h>
#include "winsupport.hpp"
#endif

#include <algorithm>
#include <cinttypes>
#include <cstdarg>
#include <cstdio>
#include <ctime>
#include <functional>
#include <list>
#include <map>
#include <regex>
#include <string>

#include "logvisor/logvisor.hpp"
#include "athena/Global.hpp"
#include "../extern/boo/xxhash/xxhash.h"
#include "SystemChar.hpp"
#include "FourCC.hpp"

#if defined(__has_feature)
#if __has_feature(thread_sanitizer)
#define HECL_NO_SANITIZE_THREAD __attribute__((no_sanitize("thread")))
#endif
#endif
#ifndef HECL_NO_SANITIZE_THREAD
#define HECL_NO_SANITIZE_THREAD
#endif

namespace hecl {
namespace Database {
class Project;
struct DataSpecEntry;
} // namespace Database

namespace blender {
enum class BlendType { None, Mesh, ColMesh, Actor, Area, World, MapArea, MapUniverse, Frame, PathMesh };

class Connection;
class Token;
class DataStream;
class PyOutStream;
class ANIMOutStream;
struct Mesh;
struct Material;
struct ColMesh;
struct World;
struct Light;
struct MapArea;
struct MapUniverse;
struct Actor;
struct Armature;
struct Action;
struct Bone;
struct PathMesh;
struct Matrix3f;
struct Matrix4f;
struct PoolSkinIndex;

extern class Token SharedBlenderToken;
} // namespace blender

extern unsigned VerbosityLevel;
extern bool GuiMode;
extern logvisor::Module LogModule;

std::string WideToUTF8(std::wstring_view src);
std::string Char16ToUTF8(std::u16string_view src);
std::wstring Char16ToWide(std::u16string_view src);
std::wstring UTF8ToWide(std::string_view src);
std::u16string UTF8ToChar16(std::string_view src);

/* humanize_number port from FreeBSD's libutil */
enum class HNFlags { None = 0, Decimal = 0x01, NoSpace = 0x02, B = 0x04, Divisor1000 = 0x08, IECPrefixes = 0x10 };
ENABLE_BITWISE_ENUM(HNFlags)

enum class HNScale { None = 0, AutoScale = 0x20 };
ENABLE_BITWISE_ENUM(HNScale)

std::string HumanizeNumber(int64_t quotient, size_t len, const char* suffix, int scale, HNFlags flags);

#if HECL_UCS2
class SystemUTF8Conv {
  std::string m_utf8;

public:
  explicit SystemUTF8Conv(SystemStringView str) : m_utf8(WideToUTF8(str)) {}
  std::string_view str() const { return m_utf8; }
  const char* c_str() const { return m_utf8.c_str(); }
  std::string operator+(std::string_view other) const { return m_utf8 + other.data(); }
};
inline std::string operator+(std::string_view lhs, const SystemUTF8Conv& rhs) { return std::string(lhs) + rhs.c_str(); }
class SystemStringConv {
  std::wstring m_sys;

public:
  explicit SystemStringConv(std::string_view str) : m_sys(UTF8ToWide(str)) {}
  SystemStringView sys_str() const { return m_sys; }
  const SystemChar* c_str() const { return m_sys.c_str(); }
  std::wstring operator+(const std::wstring_view other) const { return m_sys + other.data(); }
};
inline std::wstring operator+(std::wstring_view lhs, const SystemStringConv& rhs) {
  return std::wstring(lhs) + rhs.c_str();
}
#else
class SystemUTF8Conv {
  std::string_view m_utf8;

public:
  explicit SystemUTF8Conv(SystemStringView str) : m_utf8(str) {}
  std::string_view str() const { return m_utf8; }
  const char* c_str() const { return m_utf8.data(); }
  std::string operator+(std::string_view other) const { return std::string(m_utf8) + other.data(); }
};
inline std::string operator+(std::string_view lhs, const SystemUTF8Conv& rhs) { return std::string(lhs) + rhs.c_str(); }
class SystemStringConv {
  std::string_view m_sys;

public:
  explicit SystemStringConv(std::string_view str) : m_sys(str) {}
  SystemStringView sys_str() const { return m_sys; }
  const SystemChar* c_str() const { return m_sys.data(); }
  std::string operator+(std::string_view other) const { return std::string(m_sys) + other.data(); }
};
inline std::string operator+(std::string_view lhs, const SystemStringConv& rhs) {
  return std::string(lhs) + rhs.c_str();
}
#endif

void SanitizePath(std::string& path);
void SanitizePath(std::wstring& path);

inline void Unlink(const SystemChar* file) {
#if _WIN32
  _wunlink(file);
#else
  unlink(file);
#endif
}

inline void MakeDir(const char* dir) {
#if _WIN32
  HRESULT err;
  if (!CreateDirectoryA(dir, NULL))
    if ((err = GetLastError()) != ERROR_ALREADY_EXISTS)
      LogModule.report(logvisor::Fatal, fmt("MakeDir(%s)"), dir);
#else
  if (mkdir(dir, 0755))
    if (errno != EEXIST)
      LogModule.report(logvisor::Fatal, fmt("MakeDir({}): {}"), dir, strerror(errno));<commit_msg>hecl/hecl: Slightly improve resource usage within string conv operator+ funcs<commit_after>#pragma once

#ifndef _WIN32
#include <cstdlib>
#include <sys/stat.h>
#include <sys/file.h>
#include <sys/ioctl.h>
#include <dirent.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/statvfs.h>
#if __linux__ || __APPLE__
extern "C" int rep_closefrom(int lower);
#define closefrom rep_closefrom
#endif
#else
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN 1
#endif
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <Windows.h>
#include <cwchar>
#include <cwctype>
#include <Shlwapi.h>
#include "winsupport.hpp"
#endif

#include <algorithm>
#include <cinttypes>
#include <cstdarg>
#include <cstdio>
#include <ctime>
#include <functional>
#include <list>
#include <map>
#include <regex>
#include <string>

#include "logvisor/logvisor.hpp"
#include "athena/Global.hpp"
#include "../extern/boo/xxhash/xxhash.h"
#include "SystemChar.hpp"
#include "FourCC.hpp"

#if defined(__has_feature)
#if __has_feature(thread_sanitizer)
#define HECL_NO_SANITIZE_THREAD __attribute__((no_sanitize("thread")))
#endif
#endif
#ifndef HECL_NO_SANITIZE_THREAD
#define HECL_NO_SANITIZE_THREAD
#endif

namespace hecl {
namespace Database {
class Project;
struct DataSpecEntry;
} // namespace Database

namespace blender {
enum class BlendType { None, Mesh, ColMesh, Actor, Area, World, MapArea, MapUniverse, Frame, PathMesh };

class Connection;
class Token;
class DataStream;
class PyOutStream;
class ANIMOutStream;
struct Mesh;
struct Material;
struct ColMesh;
struct World;
struct Light;
struct MapArea;
struct MapUniverse;
struct Actor;
struct Armature;
struct Action;
struct Bone;
struct PathMesh;
struct Matrix3f;
struct Matrix4f;
struct PoolSkinIndex;

extern class Token SharedBlenderToken;
} // namespace blender

extern unsigned VerbosityLevel;
extern bool GuiMode;
extern logvisor::Module LogModule;

std::string WideToUTF8(std::wstring_view src);
std::string Char16ToUTF8(std::u16string_view src);
std::wstring Char16ToWide(std::u16string_view src);
std::wstring UTF8ToWide(std::string_view src);
std::u16string UTF8ToChar16(std::string_view src);

/* humanize_number port from FreeBSD's libutil */
enum class HNFlags { None = 0, Decimal = 0x01, NoSpace = 0x02, B = 0x04, Divisor1000 = 0x08, IECPrefixes = 0x10 };
ENABLE_BITWISE_ENUM(HNFlags)

enum class HNScale { None = 0, AutoScale = 0x20 };
ENABLE_BITWISE_ENUM(HNScale)

std::string HumanizeNumber(int64_t quotient, size_t len, const char* suffix, int scale, HNFlags flags);

#if HECL_UCS2
class SystemUTF8Conv {
  std::string m_utf8;

public:
  explicit SystemUTF8Conv(SystemStringView str) : m_utf8(WideToUTF8(str)) {}
  std::string_view str() const { return m_utf8; }
  const char* c_str() const { return m_utf8.c_str(); }
  std::string operator+(std::string_view other) const { return m_utf8 + other.data(); }
};
inline std::string operator+(std::string_view lhs, const SystemUTF8Conv& rhs) {
  return std::string(lhs).append(rhs.str());
}
class SystemStringConv {
  std::wstring m_sys;

public:
  explicit SystemStringConv(std::string_view str) : m_sys(UTF8ToWide(str)) {}
  SystemStringView sys_str() const { return m_sys; }
  const SystemChar* c_str() const { return m_sys.c_str(); }
  std::wstring operator+(const std::wstring_view other) const { return m_sys + other.data(); }
};
inline std::wstring operator+(std::wstring_view lhs, const SystemStringConv& rhs) {
  return std::wstring(lhs).append(rhs.sys_str());
}
#else
class SystemUTF8Conv {
  std::string_view m_utf8;

public:
  explicit SystemUTF8Conv(SystemStringView str) : m_utf8(str) {}
  std::string_view str() const { return m_utf8; }
  const char* c_str() const { return m_utf8.data(); }
  std::string operator+(std::string_view other) const { return std::string(m_utf8).append(other); }
};
inline std::string operator+(std::string_view lhs, const SystemUTF8Conv& rhs) {
  return std::string(lhs).append(rhs.str());
}
class SystemStringConv {
  std::string_view m_sys;

public:
  explicit SystemStringConv(std::string_view str) : m_sys(str) {}
  SystemStringView sys_str() const { return m_sys; }
  const SystemChar* c_str() const { return m_sys.data(); }
  std::string operator+(std::string_view other) const { return std::string(m_sys).append(other); }
};
inline std::string operator+(std::string_view lhs, const SystemStringConv& rhs) {
  return std::string(lhs).append(rhs.sys_str());
}
#endif

void SanitizePath(std::string& path);
void SanitizePath(std::wstring& path);

inline void Unlink(const SystemChar* file) {
#if _WIN32
  _wunlink(file);
#else
  unlink(file);
#endif
}

inline void MakeDir(const char* dir) {
#if _WIN32
  HRESULT err;
  if (!CreateDirectoryA(dir, NULL))
    if ((err = GetLastError()) != ERROR_ALREADY_EXISTS)
      LogModule.report(logvisor::Fatal, fmt("MakeDir(%s)"), dir);
#else
  if (mkdir(dir, 0755))
    if (errno != EEXIST)
      LogModule.report(logvisor::Fatal, fmt("MakeDir({}): {}"), dir, strerror(errno));
{'fraction_non_alphanumeric': 0.1315093572009764, 'fraction_numerical': 0.015866558177379985, 'mean_word_length': 6.3765941485371345, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '327953', 'n_tokens_mistral': 3611, 'n_tokens_neox': 3501, 'n_words': 872}