<commit_before>#include "zip_folder_accessor.hpp"

#include <array>
#include <memory>

#include "nova_renderer/util/logger.hpp"
#include "nova_renderer/util/utils.hpp"

namespace nova::renderer {
    ZipFolderAccessor::ZipFolderAccessor(const fs::path& folder) : FolderAccessorBase(folder), files(new FileTreeNode) {
        const auto folder_string = folder.string();

        if(mz_zip_reader_init_file(&zip_archive, folder_string.c_str(), 0) == 0) {
            NOVA_LOG(ERROR) << "Could not open zip archive " << folder_string;
        }

        build_file_tree();
    }

    ZipFolderAccessor::~ZipFolderAccessor() {
        mz_zip_reader_end(&zip_archive);
        delete_file_tree(files);
    }

    void ZipFolderAccessor::delete_file_tree(std::unique_ptr<FileTreeNode>& node) { node = nullptr; }

    std::string ZipFolderAccessor::read_text_file(const fs::path& resource_path) {
        const fs::path full_path = *root_folder / resource_path;

        const std::string resource_string = full_path.string();
        if(!does_resource_exist_on_filesystem(full_path)) {
            NOVA_LOG(ERROR) << "Resource at path " << resource_string << " does not exist";
        }

        const uint32_t file_idx = resource_indexes.at(resource_string);

        mz_zip_archive_file_stat file_stat = {};
        const mz_bool has_file_stat = mz_zip_reader_file_stat(&zip_archive, file_idx, &file_stat);
        if(has_file_stat == 0) {
            const mz_zip_error err_code = mz_zip_get_last_error(&zip_archive);
            const std::string err = mz_zip_get_error_string(err_code);

            NOVA_LOG(ERROR) << "Could not get information for file " << resource_string << ": " << err;
        }

        std::pmr::vector<char> resource_buffer;
        resource_buffer.reserve(static_cast<uint64_t>(file_stat.m_uncomp_size));

        const mz_bool file_extracted = mz_zip_reader_extract_to_mem(&zip_archive,
                                                                    file_idx,
                                                                    resource_buffer.data(),
                                                                    resource_buffer.size(),
                                                                    0);
        if(file_extracted == 0) {
            const mz_zip_error err_code = mz_zip_get_last_error(&zip_archive);
            const std::string err = mz_zip_get_error_string(err_code);

            NOVA_LOG(ERROR) << "Could not extract file " << resource_string << ": " << err;
        }

        return std::string{resource_buffer.data()};
    }

    std::pmr::vector<fs::path> ZipFolderAccessor::get_all_items_in_folder(const fs::path& folder) {
        const std::string folder_stringname = folder.string().c_str();
        std::pmr::vector<std::string> folder_path_parts = split(folder_stringname, '/');

        FileTreeNode* cur_node = files.get();
        // Get the node at this path
        for(const std::string& part : folder_path_parts) {
            bool found_node = false;
            for(std::unique_ptr<FileTreeNode>& child : cur_node->children) {
                if(child->name == part) {
                    cur_node = child.get();
                    found_node = true;
                    break;
                }
            }
<commit_msg>[loading] More better string usage<commit_after>#include "zip_folder_accessor.hpp"

#include <array>
#include <memory>

#include "nova_renderer/util/logger.hpp"
#include "nova_renderer/util/utils.hpp"

namespace nova::renderer {
    ZipFolderAccessor::ZipFolderAccessor(const fs::path& folder) : FolderAccessorBase(folder), files(new FileTreeNode) {
        const auto folder_string = folder.string();

        if(mz_zip_reader_init_file(&zip_archive, folder_string.c_str(), 0) == 0) {
            NOVA_LOG(ERROR) << "Could not open zip archive " << folder_string;
        }

        build_file_tree();
    }

    ZipFolderAccessor::~ZipFolderAccessor() {
        mz_zip_reader_end(&zip_archive);
        delete_file_tree(files);
    }

    void ZipFolderAccessor::delete_file_tree(std::unique_ptr<FileTreeNode>& node) { node = nullptr; }

    std::string ZipFolderAccessor::read_text_file(const fs::path& resource_path) {
        const fs::path full_path = *root_folder / resource_path;

        const std::string resource_string = full_path.string();
        if(!does_resource_exist_on_filesystem(full_path)) {
            NOVA_LOG(ERROR) << "Resource at path " << resource_string << " does not exist";
        }

        const uint32_t file_idx = resource_indexes.at(resource_string);

        mz_zip_archive_file_stat file_stat = {};
        const mz_bool has_file_stat = mz_zip_reader_file_stat(&zip_archive, file_idx, &file_stat);
        if(has_file_stat == 0) {
            const mz_zip_error err_code = mz_zip_get_last_error(&zip_archive);
            const std::string err = mz_zip_get_error_string(err_code);

            NOVA_LOG(ERROR) << "Could not get information for file " << resource_string << ": " << err;
        }

        std::pmr::vector<char> resource_buffer;
        resource_buffer.reserve(static_cast<uint64_t>(file_stat.m_uncomp_size));

        const mz_bool file_extracted = mz_zip_reader_extract_to_mem(&zip_archive,
                                                                    file_idx,
                                                                    resource_buffer.data(),
                                                                    resource_buffer.size(),
                                                                    0);
        if(file_extracted == 0) {
            const mz_zip_error err_code = mz_zip_get_last_error(&zip_archive);
            const std::string err = mz_zip_get_error_string(err_code);

            NOVA_LOG(ERROR) << "Could not extract file " << resource_string << ": " << err;
        }

        return std::string{resource_buffer.data()};
    }

    std::pmr::vector<fs::path> ZipFolderAccessor::get_all_items_in_folder(const fs::path& folder) {
        const std::string folder_stringname = folder.string();
        std::pmr::vector<std::string> folder_path_parts = split(folder_stringname, '/');

        FileTreeNode* cur_node = files.get();
        // Get the node at this path
        for(const std::string& part : folder_path_parts) {
            bool found_node = false;
            for(std::unique_ptr<FileTreeNode>& child : cur_node->children) {
                if(child->name == part) {
                    cur_node = child.get();
                    found_node = true;
                    break;
                }
            }

{'fraction_non_alphanumeric': 0.10662041924294978, 'fraction_numerical': 0.0027145226964258784, 'mean_word_length': 2.257367387033399, 'pattern_counts': {'":': 4, '<': 71, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3086786', 'n_tokens_mistral': 1939, 'n_tokens_neox': 1877, 'n_words': 328}