<commit_before>#include "file_io.h"
#include <cstdlib>
#include <cstdio>
#include <cerrno>
#include <array>
#include <sys/stat.h> // stat,fstat
#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING 1
#include <codecvt> // codecvt_utf8
#include <locale>  // wstring_convert
#if _WIN32
    #define WIN32_LEAN_AND_MEAN
    #define _CRT_DISABLE_PERFCRIT_LOCKS 1 // we're running single-threaded I/O only
    #include <Windows.h>
    #include <direct.h> // mkdir, getcwd
    #include <io.h>     // _chsize
    #define USE_WINAPI_IO 1
    #define stat64 _stat64
    #define fseeki64 _fseeki64
    #define ftelli64 _ftelli64
#else
    #include <unistd.h>
    #include <dirent.h> // opendir()

    #define fseeki64 fseeko
    #define ftelli64 ftello

    #define _fstat64 fstat
    #define stat64   stat
#endif
#if __APPLE__
#include <TargetConditionals.h>
#endif
#if __ANDROID__
#include <jni.h>

#endif

namespace rpp /* ReCpp */
{
    load_buffer::~load_buffer()
    {
        if (str) free(str); // MEM_RELEASE
    }
    load_buffer::load_buffer(load_buffer&& mv) noexcept : str(mv.str), len(mv.len)
    {
        mv.str = 0;
        mv.len = 0;<commit_msg>Fixed annoying min macro bug<commit_after>#include "file_io.h"
#include <cstdlib>
#include <cstdio>
#include <cerrno>
#include <array>
#include <sys/stat.h> // stat,fstat
#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING 1
#include <codecvt> // codecvt_utf8
#include <locale>  // wstring_convert
#if _WIN32
    #define WIN32_LEAN_AND_MEAN
    #define _CRT_DISABLE_PERFCRIT_LOCKS 1 // we're running single-threaded I/O only
    #include <Windows.h>
    #include <direct.h> // mkdir, getcwd
    #include <io.h>     // _chsize
    #define USE_WINAPI_IO 1
    #define stat64 _stat64
    #define fseeki64 _fseeki64
    #define ftelli64 _ftelli64
    #undef min
#else
    #include <unistd.h>
    #include <dirent.h> // opendir()

    #define fseeki64 fseeko
    #define ftelli64 ftello

    #define _fstat64 fstat
    #define stat64   stat
#endif
#if __APPLE__
#include <TargetConditionals.h>
#endif
#if __ANDROID__
#include <jni.h>

#endif

namespace rpp /* ReCpp */
{
    load_buffer::~load_buffer()
    {
        if (str) free(str); // MEM_RELEASE
    }
    load_buffer::load_buffer(load_buffer&& mv) noexcept : str(mv.str), len(mv.len)
    {
        mv.str = 0;
        mv.len = 0;
{'fraction_non_alphanumeric': 0.11732081911262798, 'fraction_numerical': 0.027303754266211604, 'mean_word_length': 3.3186003683241254, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '570131', 'n_tokens_mistral': 954, 'n_tokens_neox': 904, 'n_words': 207}