<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <errno.h>

#include <csv.h>

#include "../../readstat.h"

#include "../util/readstat_dta_days.h"
#include "json_metadata.h"
#include "read_module.h"
#include "csv_metadata.h"

#include "mod_csv.h"
#include "mod_dta.h"
#include "mod_sav.h"

#define UNUSED(x) (void)(x)

rs_read_module_t *rs_read_module_for_filename(rs_read_module_t *modules, long module_count, int output_format) {
    int i;
    for (i=0; i<module_count; i++) {
        if (modules[i].format == output_format)
            return &modules[i];
    }
    return NULL;
}

static void produce_column_header(struct csv_metadata *c, void *s, size_t len) {
    char* column = (char*)s;
    readstat_variable_t* var = &c->variables[c->columns];
    memset(var, 0, sizeof(readstat_variable_t));
    extract_metadata_type_t coltype = column_type(c->json_md, column, c->output_format);
    c->is_date[c->columns] = coltype == METADATA_COLUMN_TYPE_DATE;

    if (coltype == EXTRACT_METADATA_TYPE_STRING) {
        var->alignment = READSTAT_ALIGNMENT_LEFT;
    } else if (coltype == EXTRACT_METADATA_TYPE_NUMERIC) {
        var->alignment = READSTAT_ALIGNMENT_RIGHT;
    }

    if (c->output_module->header) {
        c->output_module->header(c, column, var);
    }

    if (c->pass == 2 && coltype == EXTRACT_METADATA_TYPE_STRING) {
        var->storage_width = c->column_width[c->columns];
    }

    var->index = c->columns;
    copy_variable_property(c->json_md, column, "label", var->label, sizeof(var->label));
    snprintf(var->name, sizeof(var->name), "%.*s", (int)len, column);

    if (c->output_module->missingness) {
        c->output_module->missingness(c, column);
    }
    if (c->output_module->value_label && c->handle.value_label) {
        c->output_module->value_label(c, column);
    }
<commit_msg>Fix compile error in the CSV reader<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <errno.h>

#include <csv.h>

#include "../../readstat.h"

#include "../util/readstat_dta_days.h"
#include "json_metadata.h"
#include "read_module.h"
#include "csv_metadata.h"

#include "mod_csv.h"
#include "mod_dta.h"
#include "mod_sav.h"

#define UNUSED(x) (void)(x)

rs_read_module_t *rs_read_module_for_filename(rs_read_module_t *modules, long module_count, int output_format) {
    int i;
    for (i=0; i<module_count; i++) {
        if (modules[i].format == output_format)
            return &modules[i];
    }
    return NULL;
}

static void produce_column_header(struct csv_metadata *c, void *s, size_t len) {
    char* column = (char*)s;
    readstat_variable_t* var = &c->variables[c->columns];
    memset(var, 0, sizeof(readstat_variable_t));
    extract_metadata_type_t coltype = column_type(c->json_md, column, c->output_format);
    c->is_date[c->columns] = coltype == EXTRACT_METADATA_TYPE_DATE;

    if (coltype == EXTRACT_METADATA_TYPE_STRING) {
        var->alignment = READSTAT_ALIGNMENT_LEFT;
    } else if (coltype == EXTRACT_METADATA_TYPE_NUMERIC) {
        var->alignment = READSTAT_ALIGNMENT_RIGHT;
    }

    if (c->output_module->header) {
        c->output_module->header(c, column, var);
    }

    if (c->pass == 2 && coltype == EXTRACT_METADATA_TYPE_STRING) {
        var->storage_width = c->column_width[c->columns];
    }

    var->index = c->columns;
    copy_variable_property(c->json_md, column, "label", var->label, sizeof(var->label));
    snprintf(var->name, sizeof(var->name), "%.*s", (int)len, column);

    if (c->output_module->missingness) {
        c->output_module->missingness(c, column);
    }
    if (c->output_module->value_label && c->handle.value_label) {
        c->output_module->value_label(c, column);
    }

{'fraction_non_alphanumeric': 0.15644253951245646, 'fraction_numerical': 0.0016072863648540047, 'mean_word_length': 3.8430609597924774, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1525105', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1364, 'n_words': 244}