<commit_before>
#include <stdio.h>
#include <stdlib.h>
#include <getopt.h>
#include <string.h>
#include <stdint.h>
#include <ctype.h>
#include <sys/stat.h>
#include <sys/types.h>

#include <hex_encode.h>
#include <util.h>

#include "trail-extractd.h"

#define DEFAULT_PORT 7676

static void show(struct extractd_ctx *ctx)
{
    static char hex_cookie[33];
    const char *cookie;
    const uint32_t *events;
    uint32_t i, j, k, num_fields, num_events;
    int ret;

    while ((ret = extractd_next_trail(ctx->extd,
                                      &cookie,
                                      &events,
                                      &num_events,
                                      &num_fields,
                                      MAPPER_TIMEOUT))){
        if (ret == -1)
            DIE("Timeout");

        hex_encode((const uint8_t*)cookie, hex_cookie);
        printf("%s", hex_cookie);
        for (i = 0, k = 0; i < num_events; i++){
            if (i > 0)
                printf(" |");
            printf(" %u", events[k++]);
            for (j = 0; j < num_fields; j++)
                printf(" %s", extractd_get_token(ctx->extd, j, events[k++]));
        }
        printf("\n");
    }
}

static void print_usage_and_exit()
{
    printf("USAGE\n");
    /* list available ops */
    exit(1);
}

const char *safe_filename(const char *value)
{
    static char safe[MAX_PATH_SIZE];
    uint32_t i;

    for (i = 0; i < MAX_PATH_SIZE - 1 && value[i]; i++)
        if (isalnum(value[i]))
            safe[i] = value[i];
        else
            safe[i] = '_';
    safe[i] = 0;

    return safe;
}

static void output_fields(const struct extractd_ctx *ctx)
{
    static char path[MAX_PATH_SIZE];
    uint32_t i, j, k;
    FILE *out;

    for (i = 0, k = 0; i < extractd_get_num_fields(ctx->extd); i++){
        const char *field = extractd_get_field_name(ctx->extd, i);
        const char *safe = safe_filename(field);

        if (ctx->groupby_str && !strcmp(field, ctx->groupby_str))
            continue;

        if (ctx->dir)
            make_path(path, "%s/trail-field.%u.%s.csv", ctx->dir, k, safe);
        else
            make_path(path, "trail-field.%u.%s.csv", k, safe);

        if (!(out = fopen(path, "w")))
            DIE("Could not open output file at %s\n", path);

        for (j = 0; j < extractd_num_tokens(ctx->extd, k); j++){
            const char *token = extractd_get_token(ctx->extd, k, j);
            SAFE_FPRINTF(out, path, "%s\n", token);
        }

        ++k;
        fclose(out);
    }
}

int main(int argc, char **argv)
{
    static struct extractd_ctx ctx;
    int c, option_index = 0;

    static struct option long_options[] = {
        {"port", required_argument, 0, 'p'},
        {"show", no_argument, 0, 's'},
        {"groupby", required_argument, 0, 'g'},
        {"output-dir", required_argument, 0, 'o'},
        {0, 0, 0, 0}
    };

    do{
        c = getopt_long(argc,
                        argv,
                        "p:g:o:s",
                        long_options,<commit_msg>fix csv output in trail-extractd<commit_after>
#include <stdio.h>
#include <stdlib.h>
#include <getopt.h>
#include <string.h>
#include <stdint.h>
#include <ctype.h>
#include <sys/stat.h>
#include <sys/types.h>

#include <hex_encode.h>
#include <util.h>

#include "trail-extractd.h"

#define DEFAULT_PORT 7676

static void show(struct extractd_ctx *ctx)
{
    static char hex_cookie[33];
    const char *cookie;
    const uint32_t *events;
    uint32_t i, j, k, num_fields, num_events;
    int ret;

    while ((ret = extractd_next_trail(ctx->extd,
                                      &cookie,
                                      &events,
                                      &num_events,
                                      &num_fields,
                                      MAPPER_TIMEOUT))){
        if (ret == -1)
            DIE("Timeout");

        hex_encode((const uint8_t*)cookie, hex_cookie);
        printf("%s", hex_cookie);
        for (i = 0, k = 0; i < num_events; i++){
            if (i > 0)
                printf(" |");
            printf(" %u", events[k++]);
            for (j = 0; j < num_fields; j++)
                printf(" %s", extractd_get_token(ctx->extd, j, events[k++]));
        }
        printf("\n");
    }
}

static void print_usage_and_exit()
{
    printf("USAGE\n");
    /* list available ops */
    exit(1);
}

const char *safe_filename(const char *value)
{
    static char safe[MAX_PATH_SIZE];
    uint32_t i;

    for (i = 0; i < MAX_PATH_SIZE - 1 && value[i]; i++)
        if (isalnum(value[i]))
            safe[i] = value[i];
        else
            safe[i] = '_';
    safe[i] = 0;

    return safe;
}

static void output_fields(const struct extractd_ctx *ctx)
{
    static char path[MAX_PATH_SIZE];
    uint32_t i, j, k;
    FILE *out;

    for (i = 0, k = 0; i < extractd_get_num_fields(ctx->extd); i++){
        const char *field = extractd_get_field_name(ctx->extd, i);
        const char *safe = safe_filename(field);

        if (ctx->groupby_str && !strcmp(field, ctx->groupby_str))
            continue;

        if (ctx->dir)
            make_path(path, "%s/trail-field.%u.%s.csv", ctx->dir, k, safe);
        else
            make_path(path, "trail-field.%u.%s.csv", k, safe);

        if (!(out = fopen(path, "w")))
            DIE("Could not open output file at %s\n", path);

        for (j = 0; j < extractd_num_tokens(ctx->extd, i); j++){
            const char *token = extractd_get_token(ctx->extd, i, j);
            SAFE_FPRINTF(out, path, "%s\n", token);
        }

        ++k;
        fclose(out);
    }
}

int main(int argc, char **argv)
{
    static struct extractd_ctx ctx;
    int c, option_index = 0;

    static struct option long_options[] = {
        {"port", required_argument, 0, 'p'},
        {"show", no_argument, 0, 's'},
        {"groupby", required_argument, 0, 'g'},
        {"output-dir", required_argument, 0, 'o'},
        {0, 0, 0, 0}
    };

    do{
        c = getopt_long(argc,
                        argv,
                        "p:g:o:s",
                        long_options,
{'fraction_non_alphanumeric': 0.15157448197095774, 'fraction_numerical': 0.011747430249632892, 'mean_word_length': 1.8067765567765568, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3495019', 'n_tokens_mistral': 2165, 'n_tokens_neox': 2057, 'n_words': 505}