<commit_before>#include "DataStore/LocalFileDataStore.h"

#include "FileSystem/FileSystem.h"

namespace Flourish
{
    LocalFileDataStore::LocalFileDataStore(const char* root)
        : _root(root)
    {
        FileSystem::CreateDirectoryTree(root);
        auto lastChar = root[_root.length() - 1];
        if (lastChar != '\\' && lastChar != '/')
        {
            _root.append("/");
        }
    }

    LocalFileDataStore::~LocalFileDataStore()
    {
        for (auto iter : _pathToOpenFile)
        {
            delete iter.second;
        }
    }

    bool LocalFileDataStore::Exists(const DataStorePath& path) const
    {
        auto pathAsCStr = GetFullPath(path).c_str();
        return FileSystem::FileExists(pathAsCStr) || FileSystem::DirExists(pathAsCStr);
    }

    void LocalFileDataStore::OpenForRead(const DataStorePath& path, DataStoreReadCallback callback)
    {
        if (_pathToOpenFile.find(path) == _pathToOpenFile.end())
        {
            DataBuffer buffer(1024);
            auto file = FileSystem::OpenRead(GetFullPath(path).c_str());
            if (file == nullptr)
            {
                std::string errorMessage = "Path '";
                errorMessage.append(path.AsString());
                errorMessage.append("' does not exist in data store");
                callback(DataStoreReadCallbackParam::Failure(errorMessage.c_str()));
                return;
            }
            FileSystem::Read(static_cast<uint8_t*>(buffer.WriteData()), buffer.Size(), file);
            const auto stream = std::make_shared<DataStoreReadStream>(this, path, buffer);
            _pathToOpenFile.insert({path, new OpenFile(file, stream)});
            callback(DataStoreReadCallbackParam::Successful(stream));
        }
        else
        {
            std::string error("Path: '");
            error.append(path.AsString());<commit_msg>Fixed bug in local data store which causes char array to be free'd under user<commit_after>#include "DataStore/LocalFileDataStore.h"

#include "FileSystem/FileSystem.h"

namespace Flourish
{
    LocalFileDataStore::LocalFileDataStore(const char* root)
        : _root(root)
    {
        FileSystem::CreateDirectoryTree(root);
        auto lastChar = root[_root.length() - 1];
        if (lastChar != '\\' && lastChar != '/')
        {
            _root.append("/");
        }
    }

    LocalFileDataStore::~LocalFileDataStore()
    {
        for (auto iter : _pathToOpenFile)
        {
            delete iter.second;
        }
    }

    bool LocalFileDataStore::Exists(const DataStorePath& path) const
    {
		auto fullPath = GetFullPath(path);
        auto pathAsCStr = fullPath.c_str();
        return FileSystem::FileExists(pathAsCStr) || FileSystem::DirExists(pathAsCStr);
    }

    void LocalFileDataStore::OpenForRead(const DataStorePath& path, DataStoreReadCallback callback)
    {
        if (_pathToOpenFile.find(path) == _pathToOpenFile.end())
        {
            DataBuffer buffer(1024);
            auto file = FileSystem::OpenRead(GetFullPath(path).c_str());
            if (file == nullptr)
            {
                std::string errorMessage = "Path '";
                errorMessage.append(path.AsString());
                errorMessage.append("' does not exist in data store");
                callback(DataStoreReadCallbackParam::Failure(errorMessage.c_str()));
                return;
            }
            FileSystem::Read(static_cast<uint8_t*>(buffer.WriteData()), buffer.Size(), file);
            const auto stream = std::make_shared<DataStoreReadStream>(this, path, buffer);
            _pathToOpenFile.insert({path, new OpenFile(file, stream)});
            callback(DataStoreReadCallbackParam::Successful(stream));
        }
        else
        {
            std::string error("Path: '");
            error.append(path.AsString());
{'fraction_non_alphanumeric': 0.12181818181818181, 'fraction_numerical': 0.003116883116883117, 'mean_word_length': 2.3929515418502203, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3742240', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1058, 'n_words': 186}