<commit_before>//
// Created by Bart on 4/21/2015.
//

#include "FileSystem.h"

FileSystem::FileSystem() {
    // Stub
}

FileSystem::~FileSystem() {
    // Stub
}

FSResult FileSystem::loadDirectory(char *directory) {
    // Does this directory exist... or is it even a directory?
    if (!exists(directory) || !is_directory(directory)) {
        return FileSystem::E_INVALID_DIRECTORY;
    }

    // Discover main_file_cache.dat2 (the '/' operator glues two paths)
    path mainFile = path(directory) / string("main_file_cache.dat2");
    if (!filesystem::exists(mainFile)) {
        return FileSystem::E_NO_MAINFILE;
    }

    // Load all the *.idx files
    map<int, path*> validIndices; // Map to hold our .idx file paths
    for (directory_entry e : make_iterator_range(directory_iterator(directory))) { // Loop
        if (starts_with(e.path().extension().string(), ".idx") && e.path().string().size() > 4) { // Is this an index descriptor?
            int idxId = atoi(e.path().extension().string().substr(4).c_str()); // Substring from 4, parse as integer.

            // Check if the range is valid. If so, store it in our map.
            if (idxId >= 0 && idxId <= 255) {
                validIndices[idxId] = new path(e); // Copy, otherwise it falls out of scope
            }
        }
    }

    // Set this instance's map to the allocated one
    this->validIndices = validIndices;


    return FileSystem::RESULT_OK; // All good.
}<commit_msg>Make code less verbose<commit_after>//
// Created by Bart on 4/21/2015.
//

#include "FileSystem.h"

FileSystem::FileSystem() {
    // Stub
}

FileSystem::~FileSystem() {
    // Stub
}

FSResult FileSystem::loadDirectory(char *directory) {
    // Does this directory exist... or is it even a directory?
    if (!exists(directory) || !is_directory(directory)) {
        return FileSystem::E_INVALID_DIRECTORY;
    }

    // Discover main_file_cache.dat2 (the '/' operator glues two paths)
    path mainFile = path(directory) / string("main_file_cache.dat2");
    if (!exists(mainFile)) {
        return E_NO_MAINFILE;
    }

    // Load all the *.idx files
    map<int, path*> validIndices; // Map to hold our .idx file paths
    for (directory_entry e : make_iterator_range(directory_iterator(directory))) { // Loop
        path current = e.path();
        string extension = current.extension().string();

        if (starts_with(extension, ".idx") && extension.size() > 4) { // Is this an index descriptor?
            int idxId = atoi(extension.substr(4).c_str()); // Substring from 4, parse as integer.

            // Check if the range is valid. If so, store it in our map.
            if (idxId >= 0 && idxId <= 255) {
                validIndices[idxId] = new path(current); // Copy, otherwise it falls out of scope
            }
        }
    }

    // Set this instance's map to the allocated one
    this->validIndices = validIndices;


    return FileSystem::RESULT_OK; // All good.
}
{'fraction_non_alphanumeric': 0.13556457065584854, 'fraction_numerical': 0.010818120351588911, 'mean_word_length': 2.582324455205811, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4075321', 'n_tokens_mistral': 914, 'n_tokens_neox': 876, 'n_words': 261}