<commit_before>#include "filestore.h"

#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>

#define _err(...) do { \
    fprintf(stderr, __VA_ARGS__); \
    fprintf(stderr, "\n"); \
} while (0)

struct filestate {
    FILE *f;
    char *data;
    size_t len;
    struct stat stat;
    size_t start, end;
};

static const char* _chunker(void *data, unsigned long offset, size_t count)
{
    struct filestate *state = data;

    /*
     * This code could be optimized, as it might read the same data more than
     * once. However, we know that the particular usage of this program will
     * read data unidirectionally in small chunks, so optimization and / or
     * generalization are probably unwarranted.
     */

    if (offset < state->start || offset + count >= state->end) {
        if (count > state->len) {
            state->data = realloc(state->data, state->len = count * 2);
        }

        fseek(state->f, state->start = offset, SEEK_SET);
        state->end = state->start + fread(state->data, 1, state->len, state->f);
    }

    return &state->data[offset - state->start];
}

int hd_read_file_init(struct hd_parser_state *parser_state, void *userdata)
{
    int rc = 0;

    const char *filename = userdata;

    struct filestate *state = malloc(sizeof *state);

    state->f = fopen(filename, "r");
    if (!state->f) {
        _err("File '%s' could not be opened (%d: %s)",
             filename, errno, strerror(errno));
        return -1;
    }

    rc = stat(filename, &state->stat);
    if (rc) {
        _err("stat: %d: %s", errno, strerror(errno));
        return rc;
    }

    state->len = BUFSIZ;
    state->data = malloc(state->len);
    state->start = state->end = 0;

    hd_set_userdata(parser_state, state);
    hd_set_chunker(parser_state, _chunker);

    return rc;
}

int hd_read_file_fini(struct hd_parser_state *parser_state)
{
    struct filestate *state = hd_get_userdata(parser_state);

    if (!state) return -1;

    fclose(state->f);
    free(state);

    return 0;
}

/* vim:set ts=4 sw=4 syntax=c.doxygen: */<commit_msg>Error checking; filename "-" works for stdin<commit_after>#include "filestore.h"

#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <unistd.h>

#define _err(...) do { \
    fprintf(stderr, __VA_ARGS__); \
    fprintf(stderr, "\n"); \
} while (0)

struct filestate {
    FILE *f;
    char *data;
    size_t len;
    size_t start, end;
};

static const char* _chunker(void *data, unsigned long offset, size_t count)
{
    struct filestate *state = data;

    /*
     * This code could be optimized, as it might read the same data more than
     * once. However, we know that the particular usage of this program will
     * read data unidirectionally in small chunks, so optimization and / or
     * generalization are probably unwarranted.
     */

    if (offset < state->start || offset + count >= state->end) {
        if (count > state->len) {
            void *temp = realloc(state->data, state->len = count * 2);
            if (!temp)
                return NULL;
            state->data = temp;
        }

        if (fseek(state->f, state->start = offset, SEEK_SET))
            return NULL;
        state->end = state->start + fread(state->data, 1, state->len, state->f);
        if (state->end <= state->start)
            return NULL;
    }

    return &state->data[offset - state->start];
}

int hd_read_file_init(struct hd_parser_state *parser_state, void *userdata)
{
    int rc = 0;

    const char *filename = userdata;

    struct filestate *state = malloc(sizeof *state);

    if (!strcmp(filename, "-")) {
        state->f = stdin;
    } else {
        state->f = fopen(filename, "r");
        if (!state->f) {
            _err("File '%s' could not be opened (%d: %s)",
                 filename, errno, strerror(errno));
            return -1;
        }
    }

    state->len = BUFSIZ;
    state->data = malloc(state->len);
    state->start = state->end = 0;

    hd_set_userdata(parser_state, state);
    hd_set_chunker(parser_state, _chunker);

    return rc;
}

int hd_read_file_fini(struct hd_parser_state *parser_state)
{
    struct filestate *state = hd_get_userdata(parser_state);

    if (!state) return -1;

    if (state->f != stdin)
        fclose(state->f);
    free(state);

    return 0;
}

/* vim:set ts=4 sw=4 syntax=c.doxygen: */
{'fraction_non_alphanumeric': 0.141720097280566, 'fraction_numerical': 0.004421843908910016, 'mean_word_length': 2.604780876494024, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7276002', 'n_tokens_mistral': 1581, 'n_tokens_neox': 1526, 'n_words': 417}