<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include "nod/Util.hpp"
#include "nod/IFileIO.hpp"

namespace nod
{

class FileIOWin32 : public IFileIO
{
    SystemString m_path;
    int64_t m_maxWriteSize;
public:
    FileIOWin32(SystemStringView path, int64_t maxWriteSize)
    : m_path(path), m_maxWriteSize(maxWriteSize) {}

    bool exists()
    {
        HANDLE fp = CreateFileW(m_path.c_str(), GENERIC_READ, FILE_SHARE_READ,
                                nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
        if (fp == INVALID_HANDLE_VALUE)
            return false;
        CloseHandle(fp);
        return true;
    }

    uint64_t size()
    {
        HANDLE fp = CreateFileW(m_path.c_str(), GENERIC_READ, FILE_SHARE_READ,
                                nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
        if (fp == INVALID_HANDLE_VALUE)
            return 0;
        LARGE_INTEGER sz;
        if (!GetFileSizeEx(fp, &sz))
        {
            CloseHandle(fp);
            return 0;
        }
        CloseHandle(fp);
        return sz.QuadPart;
    }

    struct WriteStream : public IFileIO::IWriteStream
    {
        HANDLE fp;
        int64_t m_maxWriteSize;
        WriteStream(SystemStringView path, int64_t maxWriteSize, bool& err)
        : m_maxWriteSize(maxWriteSize)
        {
            fp = CreateFileW(path.data(), GENERIC_WRITE, FILE_SHARE_WRITE,
                             nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
            if (fp == INVALID_HANDLE_VALUE)
            {
                LogModule.report(logvisor::Error, _S("unable to open '%s' for writing"), path.data());
                err = true;
            }
        }
        WriteStream(SystemStringView path, uint64_t offset, int64_t maxWriteSize, bool& err)
        : m_maxWriteSize(maxWriteSize)
        {
            fp = CreateFileW(path.data(), GENERIC_WRITE, FILE_SHARE_WRITE,
                             nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
            if (fp == INVALID_HANDLE_VALUE)
            {
                LogModule.report(logvisor::Error, _S("unable to open '%s' for writing"), path.data());
                err = true;
                return;
            }
            LARGE_INTEGER lioffset;
            lioffset.QuadPart = offset;
            SetFilePointerEx(fp, lioffset, nullptr, FILE_BEGIN);
        }
        ~WriteStream()
        {
            CloseHandle(fp);
        }
        uint64_t write(const void* buf, uint64_t length)
        {
            if (m_maxWriteSize >= 0)
            {
                LARGE_INTEGER li = {};
                LARGE_INTEGER res;
                SetFilePointerEx(fp, li, &res, FILE_CURRENT);
                if (res.QuadPart + int64_t(length) > m_maxWriteSize)
                {
                    LogModule.report(logvisor::Error, _S("write operation exceeds file's %" PRIi64 "-byte limit"), m_maxWriteSize);
                    return 0;
                }
            }

            DWORD ret = 0;
            WriteFile(fp, buf, length, &ret, nullptr);
            return ret;
        }
    };
    std::unique_ptr<IWriteStream> beginWriteStream() const
    {
        bool Err = false;
        auto ret = std::unique_ptr<IWriteStream>(new WriteStream(m_path, m_maxWriteSize, Err));
        if (Err)
            return {};
        return ret;
    }
    std::unique_ptr<IWriteStream> beginWriteStream(uint64_t offset) const
    {
        bool Err = false;
        auto ret = std::unique_ptr<IWriteStream>(new WriteStream(m_path, offset, m_maxWriteSize, Err));
        if (Err)
            return {};
        return ret;
    }

    struct ReadStream : public IFileIO::IReadStream
    {
        HANDLE fp;
        ReadStream(SystemStringView path, bool& err)
        {
            fp = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ,
                             nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
            if (fp == INVALID_HANDLE_VALUE)
            {
                err = true;
                LogModule.report(logvisor::Error, _S("unable to open '%s' for reading"), path.c_str());
            }
        }
        ReadStream(SystemStringView path, uint64_t offset, bool& err)
        : ReadStream(path, err)
        {
            if (err)
                return;
            LARGE_INTEGER lioffset;
            lioffset.QuadPart = offset;
            SetFilePointerEx(fp, lioffset, nullptr, FILE_BEGIN);
        }
        ~ReadStream()
        {
            CloseHandle(fp);
        }
        void seek(int64_t offset, int whence)
        {
            LARGE_INTEGER li;
            li.QuadPart = offset;
            SetFilePointerEx(fp, li, nullptr, whence);
        }
        uint64_t position() const
        {
            LARGE_INTEGER li = {};
            LARGE_INTEGER res;<commit_msg>Windows fixes<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include "nod/Util.hpp"
#include "nod/IFileIO.hpp"

namespace nod
{

class FileIOWin32 : public IFileIO
{
    SystemString m_path;
    int64_t m_maxWriteSize;
public:
    FileIOWin32(SystemStringView path, int64_t maxWriteSize)
    : m_path(path), m_maxWriteSize(maxWriteSize) {}

    bool exists()
    {
        HANDLE fp = CreateFileW(m_path.c_str(), GENERIC_READ, FILE_SHARE_READ,
                                nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
        if (fp == INVALID_HANDLE_VALUE)
            return false;
        CloseHandle(fp);
        return true;
    }

    uint64_t size()
    {
        HANDLE fp = CreateFileW(m_path.c_str(), GENERIC_READ, FILE_SHARE_READ,
                                nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
        if (fp == INVALID_HANDLE_VALUE)
            return 0;
        LARGE_INTEGER sz;
        if (!GetFileSizeEx(fp, &sz))
        {
            CloseHandle(fp);
            return 0;
        }
        CloseHandle(fp);
        return sz.QuadPart;
    }

    struct WriteStream : public IFileIO::IWriteStream
    {
        HANDLE fp;
        int64_t m_maxWriteSize;
        WriteStream(SystemStringView path, int64_t maxWriteSize, bool& err)
        : m_maxWriteSize(maxWriteSize)
        {
            fp = CreateFileW(path.data(), GENERIC_WRITE, FILE_SHARE_WRITE,
                             nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
            if (fp == INVALID_HANDLE_VALUE)
            {
                LogModule.report(logvisor::Error, _S("unable to open '%s' for writing"), path.data());
                err = true;
            }
        }
        WriteStream(SystemStringView path, uint64_t offset, int64_t maxWriteSize, bool& err)
        : m_maxWriteSize(maxWriteSize)
        {
            fp = CreateFileW(path.data(), GENERIC_WRITE, FILE_SHARE_WRITE,
                             nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
            if (fp == INVALID_HANDLE_VALUE)
            {
                LogModule.report(logvisor::Error, _S("unable to open '%s' for writing"), path.data());
                err = true;
                return;
            }
            LARGE_INTEGER lioffset;
            lioffset.QuadPart = offset;
            SetFilePointerEx(fp, lioffset, nullptr, FILE_BEGIN);
        }
        ~WriteStream()
        {
            CloseHandle(fp);
        }
        uint64_t write(const void* buf, uint64_t length)
        {
            if (m_maxWriteSize >= 0)
            {
                LARGE_INTEGER li = {};
                LARGE_INTEGER res;
                SetFilePointerEx(fp, li, &res, FILE_CURRENT);
                if (res.QuadPart + int64_t(length) > m_maxWriteSize)
                {
                    LogModule.report(logvisor::Error, _S("write operation exceeds file's %" PRIi64 "-byte limit"), m_maxWriteSize);
                    return 0;
                }
            }

            DWORD ret = 0;
            WriteFile(fp, buf, length, &ret, nullptr);
            return ret;
        }
    };
    std::unique_ptr<IWriteStream> beginWriteStream() const
    {
        bool Err = false;
        auto ret = std::unique_ptr<IWriteStream>(new WriteStream(m_path, m_maxWriteSize, Err));
        if (Err)
            return {};
        return ret;
    }
    std::unique_ptr<IWriteStream> beginWriteStream(uint64_t offset) const
    {
        bool Err = false;
        auto ret = std::unique_ptr<IWriteStream>(new WriteStream(m_path, offset, m_maxWriteSize, Err));
        if (Err)
            return {};
        return ret;
    }

    struct ReadStream : public IFileIO::IReadStream
    {
        HANDLE fp;
        ReadStream(SystemStringView path, bool& err)
        {
            fp = CreateFileW(path.data(), GENERIC_READ, FILE_SHARE_READ,
                             nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
            if (fp == INVALID_HANDLE_VALUE)
            {
                err = true;
                LogModule.report(logvisor::Error, _S("unable to open '%s' for reading"), path.data());
            }
        }
        ReadStream(SystemStringView path, uint64_t offset, bool& err)
        : ReadStream(path, err)
        {
            if (err)
                return;
            LARGE_INTEGER lioffset;
            lioffset.QuadPart = offset;
            SetFilePointerEx(fp, lioffset, nullptr, FILE_BEGIN);
        }
        ~ReadStream()
        {
            CloseHandle(fp);
        }
        void seek(int64_t offset, int whence)
        {
            LARGE_INTEGER li;
            li.QuadPart = offset;
            SetFilePointerEx(fp, li, nullptr, whence);
        }
        uint64_t position() const
        {
            LARGE_INTEGER li = {};
            LARGE_INTEGER res;
{'fraction_non_alphanumeric': 0.09432799013563502, 'fraction_numerical': 0.008014796547472256, 'mean_word_length': 1.7278587443946187, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7616919', 'n_tokens_mistral': 3001, 'n_tokens_neox': 2890, 'n_words': 600}