<commit_before>#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include "errhandle.h"
#include "oleg.h"
#include "file.h"

int _ol_get_stat(const char *filepath, struct stat *sb) {
    int fd;
    fd = open(filepath, O_RDONLY);
    if (fd == -1)
        return 0;

    int ret = 1;
    if (fstat(fd, sb) == -1)
        ret = 0;
    close(fd);
    return ret;
}

size_t _ol_get_file_size(const char *filepath) {
    struct stat sb = {0};
    int ret = _ol_get_stat(filepath, &sb);
    if (ret) /* Maybe the file doesn't exist. */
        return sb.st_size;
    return 0;
}

void *_ol_mmap(size_t to_mmap, int fd) {
    /* TODO: Investigate usage of madvise here. */
    void *to_return = NULL;

    to_return = mmap(NULL, to_mmap, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    check(to_return != MAP_FAILED, "Could not mmap file.");
    return to_return;

error:
    return NULL;
}

int _ol_open_values_with_fd(ol_database *db, const int fd, const size_t filesize) {
    const size_t to_mmap = filesize <= 0 ? VALUES_DEFAULT_SIZE : filesize;

    /* TODO: Do we need madivse(MADV_HUGEPAGE); here? */
    db->values = _ol_mmap(to_mmap, fd);
    check(db->values != NULL, "Could not mmap values file.");

    /* Make sure the file is at least as big as HASH_MALLOC */
    if (filesize == 0) {
        check(ftruncate(fd, to_mmap) != -1, "Could not truncate file for values.");
        int i;
        const int max = to_mmap / sizeof(unsigned char *);
        /* Null out the stragglers */
        for (i = 0; i < max; i++) {
            ((unsigned char **)db->values)[i] = NULL;
        }
    }

    close(fd);
    return 1;

error:
    if (fd)
        close(fd);
    return 0;
}

int _ol_ensure_values_file_size(ol_database *db, const size_t desired_size) {
    int values_fd = { 0 };
    char values_filename[DB_NAME_SIZE] = { 0 };

    /* Figure out the filename */
    db->get_db_file_name(db, VALUES_FILENAME, values_filename);
    size_t filesize = _ol_get_file_size(values_filename);
    const size_t size_needed = db->val_size + desired_size;

    if (filesize > size_needed)
        return 1;
<commit_msg>Switch other check as well. Updated comment.<commit_after>#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include "errhandle.h"
#include "oleg.h"
#include "file.h"

int _ol_get_stat(const char *filepath, struct stat *sb) {
    int fd;
    fd = open(filepath, O_RDONLY);
    if (fd == -1)
        return 0;

    int ret = 1;
    if (fstat(fd, sb) == -1)
        ret = 0;
    close(fd);
    return ret;
}

size_t _ol_get_file_size(const char *filepath) {
    struct stat sb = {0};
    int ret = _ol_get_stat(filepath, &sb);
    if (ret) /* Maybe the file doesn't exist. */
        return sb.st_size;
    return 0;
}

void *_ol_mmap(size_t to_mmap, int fd) {
    /* TODO: Investigate usage of madvise here. */
    void *to_return = NULL;

    to_return = mmap(NULL, to_mmap, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    check(to_return != MAP_FAILED, "Could not mmap file.");
    return to_return;

error:
    return NULL;
}

int _ol_open_values_with_fd(ol_database *db, const int fd, const size_t filesize) {
    const size_t to_mmap = filesize == 0 ? VALUES_DEFAULT_SIZE : filesize;

    /* TODO: Do we need madivse(MADV_HUGEPAGE); here? */
    db->values = _ol_mmap(to_mmap, fd);
    check(db->values != NULL, "Could not mmap values file.");

    /* Make sure the file is at least as big as VALUES_DEFAULT_SIZE. This
     * should only happen on init. */
    if (filesize == 0) {
        check(ftruncate(fd, to_mmap) != -1, "Could not truncate file for values.");
        int i;
        const int max = to_mmap / sizeof(unsigned char *);
        /* Null out the stragglers */
        for (i = 0; i < max; i++) {
            ((unsigned char **)db->values)[i] = NULL;
        }
    }

    close(fd);
    return 1;

error:
    if (fd)
        close(fd);
    return 0;
}

int _ol_ensure_values_file_size(ol_database *db, const size_t desired_size) {
    int values_fd = { 0 };
    char values_filename[DB_NAME_SIZE] = { 0 };

    /* Figure out the filename */
    db->get_db_file_name(db, VALUES_FILENAME, values_filename);
    size_t filesize = _ol_get_file_size(values_filename);
    const size_t size_needed = db->val_size + desired_size;

    if (filesize > size_needed)
        return 1;

{'fraction_non_alphanumeric': 0.1277422303473492, 'fraction_numerical': 0.007769652650822669, 'mean_word_length': 2.7830596369922214, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7001284', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1602, 'n_words': 446}