<commit_before>/*
 * Copyright (C) 2012, 2013
 *     <NAME>
 *     <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
 * of the Software, and to permit persons to whom the Software is furnished to do
 * so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#include <string.h>
#include <stdlib.h>
#include <sys/stat.h>

#include "gmqcc.h"
#include "lexer.h"

#define HT_MACROS 1024

typedef struct {
    bool on;
    bool was_on;
    bool had_else;
} ppcondition;

typedef struct {
    int   token;
    char *value;
    /* a copy from the lexer */
    union {
        vec3_t v;
        int    i;
        double f;
        int    t; /* type */
    } constval;
} pptoken;

typedef struct {
    lex_ctx_t ctx;

    char   *name;
    char  **params;
    /* yes we need an extra flag since `#define FOO x` is not the same as `#define FOO() x` */
    bool    has_params;
    bool    variadic;

    pptoken **output;
} ppmacro;

typedef struct ftepp_s {
    lex_file    *lex;
    int          token;
    unsigned int errors;

    bool         output_on;
    ppcondition *conditions;
    /*ppmacro    **macros;*/
    ht           macros;  /* hashtable<string, ppmacro*> */
    char        *output_string;

    char        *itemname;
    char        *includename;
    bool         in_macro;

    uint32_t predef_countval;
    uint32_t predef_randval;
} ftepp_t;

/* __DATE__ */
static char *ftepp_predef_date(ftepp_t *context) {
    const struct tm *itime = NULL;
    char            *value = (char*)mem_a(82);
    time_t           rtime;

    (void)context;

    time (&rtime);
    itime = util_localtime(&rtime);
    strftime(value, 82, "\"%b %d %Y\"", itime);

    return value;
}

/* __TIME__ */
static char *ftepp_predef_time(ftepp_t *context) {
    const struct tm *itime = NULL;
    char            *value = (char*)mem_a(82);
    time_t           rtime;

    (void)context;

    time (&rtime);
    itime = util_localtime(&rtime);
    strftime(value, 82, "\"%X\"", itime);

    return value;
}

/* __LINE__ */
static char *ftepp_predef_line(ftepp_t *context) {
    char *value;

    util_asprintf(&value, "%d", (int)context->lex->line);
    return value;
}
/* __FILE__ */
static char *ftepp_predef_file(ftepp_t *context) {
    size_t length = strlen(context->lex->name) + 3; /* two quotes and a terminator */
    char  *value  = (char*)mem_a(length);

    util_snprintf(value, length, "\"%s\"", context->lex->name);
    return value;
}
/* __COUNTER_LAST__ */
static char *ftepp_predef_counterlast(ftepp_t *context) {
    char *value;
    util_asprintf(&value, "%u", context->predef_countval);
    return value;
}
/* __COUNTER__ */
static char *ftepp_predef_counter(ftepp_t *context) {
    char *value;

    context->predef_countval ++;
    util_asprintf(&value, "%u", context->predef_countval);

    return value;
}
/* __RANDOM__ */
static char *ftepp_predef_random(ftepp_t *context) {
    char *value;

    context->predef_randval = (util_rand() % 0xFF) + 1;
    util_asprintf(&value, "%u", context->predef_randval);
    return value;
}
/* __RANDOM_LAST__ */
static char *ftepp_predef_randomlast(ftepp_t *context) {
    char *value;

    util_asprintf(&value, "%u", context->predef_randval);
    return value;
}
/* __TIMESTAMP__ */
static char *ftepp_predef_timestamp(ftepp_t *context) {
    struct stat finfo;
    const char *find;
    char       *value;
    size_t      size;

    if (stat(context->lex->name, &finfo))
        return util_strdup("\"<failed to determine timestamp>\"");

    find = util_ctime(&finfo.st_mtime);
    value = (char*)mem_a(strlen(find) + 1);
    memcpy(&value[1], find, (size = strlen(find)) - 1);

    value[0]    = '"';
    value[size] = '"';

    return value;
}

typedef struct {
    const char   *name;
    char       *(*func)(ftepp_t *);
} ftepp_predef_t;

static const ftepp_predef_t ftepp_predefs[] = {
    { "__LINE__",         &ftepp_predef_line        },
    { "__FILE__",         &ftepp_predef_file        },
    { "__COUNTER__",      &ftepp_predef_counter     },
    { "__COUNTER_LAST__", &ftepp_predef_counterlast },
    { "__RANDOM__",       &ftepp_predef_random      },
    { "__RANDOM_LAST__",  &ftepp_predef_randomlast  },
    { "__DATE__",         &ftepp_predef_date        },
    { "__TIME__",         &ftepp_predef_time        },
    { "__TIME_STAMP__",   &ftepp_predef_timestamp   }
};

static GMQCC_INLINE size_t ftepp_predef_index(const char *name) {
    /* no hashtable here, we simply check for one to exist the naive way */
    size_t i;
    for(i = 1; i < GMQCC_ARRAY_COUNT(ftepp_predefs) + 1; i++)
        if (!strcmp(ftepp_predefs[i-1].name, name))
            return i;
    return 0;
}

bool ftepp_predef_exists(const char *name);
bool ftepp_predef_exists(const char *name) {
    return ftepp_predef_index(name) != 0;
}

/* singleton because we're allowed */
static GMQCC_INLINE char *(*ftepp_predef(const char *name))(ftepp_t *context) {
    size_t i = ftepp_predef_index(name);
    return (i != 0) ? ftepp_predefs[i-1].func : NULL;
}

#define ftepp_tokval(f) ((f)->lex->tok.value)
#define ftepp_ctx(f)    ((f)->lex->tok.ctx)

static void ftepp_errorat(ftepp_t *ftepp, lex_ctx_t ctx, const char *fmt, ...)
{
    va_list ap;

    ftepp->errors++;

    va_start(ap, fmt);
    con_cvprintmsg(ctx, LVL_ERROR, "error", fmt, ap);
    va_end(ap);
}

static void ftepp_error(ftepp_t *ftepp, const char *fmt, ...)
{
    va_list ap;

    ftepp->errors++;

    va_start(ap, fmt);
    con_cvprintmsg(ftepp->lex->tok.ctx, LVL_ERROR, "error", fmt, ap);
    va_end(ap);
}

static bool GMQCC_WARN ftepp_warn(ftepp_t *ftepp, int warntype, const char *fmt, ...)
{
    bool    r;
    va_list ap;

    va_start(ap, fmt);
    r = vcompile_warning(ftepp->lex->tok.ctx, warntype, fmt, ap);
    va_end(ap);
    return r;
}

static pptoken *pptoken_make(ftepp_t *ftepp)
{
    pptoken *token = (pptoken*)mem_a(sizeof(pptoken));
    token->token = ftepp->token;
#if 0
    if (token->token == TOKEN_WHITE)
        token->value = util_strdup(" ");
    else
#else
        token->value = util_strdup(ftepp_tokval(ftepp));
#endif
    memcpy(&token->constval, &ftepp->lex->tok.constval, sizeof(token->constval));
    return token;
}

static GMQCC_INLINE void pptoken_delete(pptoken *self)
{
    mem_d(self->value);
    mem_d(self);
}

static ppmacro *ppmacro_new(lex_ctx_t ctx, const char *name)
{
    ppmacro *macro = (ppmacro*)mem_a(sizeof(ppmacro));

    (void)ctx;
    memset(macro, 0, sizeof(*macro));
    macro->name = util_strdup(name);
    return macro;
}

static void ppmacro_delete(ppmacro *self)
{
    size_t i;
    for (i = 0; i < vec_size(self->params); ++i)
        mem_d(self->params[i]);
    vec_free(self->params);
    for (i = 0; i < vec_size(self->output); ++i)
        pptoken_delete(self->output[i]);
    vec_free(self->output);
    mem_d(self->name);
    mem_d(self);
}

static ftepp_t* ftepp_new(void)
{
    ftepp_t *ftepp;

    ftepp = (ftepp_t*)mem_a(sizeof(*ftepp));
    memset(ftepp, 0, sizeof(*ftepp));

    ftepp->macros          = util_htnew(HT_MACROS);
    ftepp->output_on       = true;
    ftepp->predef_countval = 0;
    ftepp->predef_randval  = 0;

    return ftepp;
}

static GMQCC_INLINE void ftepp_flush_do(ftepp_t *self)
{
    vec_free(self->output_string);
}

static void ftepp_delete(ftepp_t *self)
{
    ftepp_flush_do(self);
    if (self->itemname)
        mem_d(self->itemname);
    if (self->includename)
        vec_free(self->includename);

    util_htrem(self->macros, (void (*)(void*))&ppmacro_delete);

    vec_free(self->conditions);
    if (self->lex)
        lex_close(self->lex);
    mem_d(self);
}

static void ftepp_out(ftepp_t *ftepp, const char *str, bool ignore_cond)
{
    if (ignore_cond || ftepp->output_on)
    {
        size_t len;
        char  *data;
        len = strlen(str);
        data = vec_add(ftepp->output_string, len);
        memcpy(data, str, len);
    }
}

static GMQCC_INLINE void ftepp_update_output_condition(ftepp_t *ftepp)
{
    size_t i;
    ftepp->output_on = true;
    for (i = 0; i < vec_size(ftepp->conditions); ++i)
        ftepp->output_on = ftepp->output_on && ftepp->conditions[i].on;
}

static GMQCC_INLINE ppmacro* ftepp_macro_find(ftepp_t *ftepp, const char *name)
{
    return (ppmacro*)util_htget(ftepp->macros, name);
}

static GMQCC_INLINE void ftepp_macro_delete(ftepp_t *ftepp, const char *name)
{
    util_htrm(ftepp->macros, name, (void (*)(void*))&ppmacro_delete);
}

static GMQCC_INLINE int ftepp_next(ftepp_t *ftepp)
{
    return (ftepp->token = lex_do(ftepp->lex));
}

/* Important: this does not skip newlines! */
static bool ftepp_skipspace(ftepp_t *ftepp)
{
    if (ftepp->token != TOKEN_WHITE)
        return true;
    while (ftepp_next(ftepp) == TOKEN_WHITE) {}
    if (ftepp->token >= TOKEN_EOF) {
        ftepp_error(ftepp, "unexpected end of preprocessor directive");
        return false;
    }
    return true;
}

/* this one skips EOLs as well */
static bool ftepp_skipallwhite(ftepp_t *ftepp)
{
    if (ftepp->token != TOKEN_WHITE && ftepp->token != TOKEN_EOL)
        return true;
    do {
        ftepp_next(ftepp);
    } while (ftepp->token == TOKEN_WHITE || ftepp->token == TOKEN_EOL);
    if (ftepp->token >= TOKEN_EOF) {
        ftepp_error(ftepp, "unexpected end of preprocessor directive");
        return false;
    }
    return true;
}

/**
 * The huge macro parsing code...
 */
static bool ftepp_define_params(ftepp_t *ftepp, ppmacro *macro)
{
    do {
        ftepp_next(ftepp);
        if (!ftepp_skipspace(ftepp))
            return false;
        if (ftepp->token == ')')
            break;
        switch (ftepp->token) {
            case TOKEN_IDENT:
            case TOKEN_TYPENAME:
            case TOKEN_KEYWORD:
                vec_push(macro->params, util_strdup(ftepp_tokval(ftepp)));
                break;
            case TOKEN_DOTS:
                macro->variadic = true;
                break;
            default:
                ftepp_error(ftepp, "unexpected token in parameter list");
                return false;
        }
        ftepp_next(ftepp);
        if (!ftepp_skipspace(ftepp))
            return false;
        if (macro->variadic && ftepp->token != ')') {
            ftepp_error(ftepp, "cannot have parameters after the variadic parameters");
            return false;
        }
    } while (ftepp->token == ',');

    if (ftepp->token != ')') {
        ftepp_error(ftepp, "expected closing paren after macro parameter list");
        return false;
    }
    ftepp_next(ftepp);
    /* skipspace happens in ftepp_define */
    return true;
}

static bool ftepp_define_body(ftepp_t *ftepp, ppmacro *macro)
{
    pptoken *ptok;
    while (ftepp->token != TOKEN_EOL && ftepp->token < TOKEN_EOF) {
        bool   subscript = false;
        size_t index     = 0;
        if (macro->variadic && !strcmp(ftepp_tokval(ftepp), "__VA_ARGS__")) {
            subscript = !!(ftepp_next(ftepp) == '#');

            if (subscript && ftepp_next(ftepp) != '#') {
                ftepp_error(ftepp, "expected `##` in __VA_ARGS__ for subscripting");
                return false;
            } else if (subscript) {
                if (ftepp_next(ftepp) == '[') {
                    if (ftepp_next(ftepp) != TOKEN_INTCONST) {
                        ftepp_error(ftepp, "expected index for __VA_ARGS__ subscript");
                        return false;
                    }

                    index = (int)strtol(ftepp_tokval(ftepp), NULL, 10);

                    if (ftepp_next(ftepp) != ']') {
                        ftepp_error(ftepp, "expected `]` in __VA_ARGS__ subscript");
                        return false;
                    }

                    /*
                     * mark it as an array to be handled later as such and not
                     * as traditional __VA_ARGS__
                     */
                    ftepp->token = TOKEN_VA_ARGS_ARRAY;
                    ptok = pptoken_make(ftepp);
                    ptok->constval.i = index;
                    vec_push(macro->output, ptok);
                    ftepp_next(ftepp);
                } else {
                    ftepp_error(ftepp, "expected `[` for subscripting of __VA_ARGS__");
                    return false;
                }
            } else {
                int old = ftepp->token;
                ftepp->token = TOKEN_VA_ARGS;
                ptok = pptoken_make(ftepp);
                vec_push(macro->output, ptok);
                ftepp->token = old;
            }
        }
        else if (macro->variadic && !strcmp(ftepp_tokval(ftepp), "__VA_COUNT__")) {
            ftepp->token = TOKEN_VA_COUNT;
            ptok         = pptoken_make(ftepp);
            vec_push(macro->output, ptok);
            ftepp_next(ftepp);
        } else {
            ptok = pptoken_make(ftepp);
            vec_push(macro->output, ptok);
            ftepp_next(ftepp);
        }
    }
    /* recursive expansion can cause EOFs here */
    if (ftepp->token != TOKEN_EOL && ftepp->token != TOKEN_EOF) {
        ftepp_error(ftepp, "unexpected junk after macro or unexpected end of file");
        return false;
    }
    return true;
}

static const char *ftepp_math_constants[][2] = {
    { "M_E",        "2.7182818284590452354"  }, /* e          */
    { "M_LOG2E",    "1.4426950408889634074"  }, /* log_2 e    */
    { "M_LOG10E",   "0.43429448190325182765" }, /* log_10 e   */
    { "M_LN2",      "0.69314718055994530942" }, /* log_e 2    */
    { "M_LN10",     "2.30258509299404568402" }, /* log_e 10   */
    { "M_PI",       "3.14159265358979323846" }, /* pi         */
    { "M_PI_2",     "1.57079632679489661923" }, /* pi/2       */
    { "M_PI_4",     "0.78539816339744830962" }, /* pi/4       */
    { "M_1_PI",     "0.31830988618379067154" }, /* 1/pi       */
    { "M_2_PI",     "0.63661977236758134308" }, /* 2/pi       */
    { "M_2_SQRTPI", "1.12837916709551257390" }, /* 2/sqrt(pi) */
    { "M_SQRT2",    "1.41421356237309504880" }, /* sqrt(2)    */
    { "M_SQRT1_2",  "0.70710678118654752440" }, /* 1/sqrt(2)  */
    { "M_TAU",      "6.28318530717958647692" }  /* pi*2       */
};

static bool ftepp_define(ftepp_t *ftepp)
{
    ppmacro *macro = NULL;
    size_t l = ftepp_ctx(ftepp).line;
    size_t i;
    bool   mathconstant = false;

    (void)ftepp_next(ftepp);
    if (!ftepp_skipspace(ftepp))
        return false;

    switch (ftepp->token) {
        case TOKEN_IDENT:
        case TOKEN_TYPENAME:
        case TOKEN_KEYWORD:
            for (i = 0; i < GMQCC_ARRAY_COUNT(ftepp_math_constants); i++) {
                if (!strcmp(ftepp_math_constants[i][0], ftepp_tokval(ftepp))) {
                    mathconstant = true;
                    break;
                }
            }

            macro = ftepp_macro_find(ftepp, ftepp_tokval(ftepp));

            /* user defined ones take precedence */
            if (macro && mathconstant) {
                ftepp_macro_delete(ftepp, ftepp_tokval(ftepp));
                macro = NULL;
            }

            if (macro && ftepp->output_on) {
                if (ftepp_warn(ftepp, WARN_CPP, "redefining `%s`", ftepp_tokval(ftepp)))
                    return false;
                ftepp_macro_delete(ftepp, ftepp_tokval(ftepp));
            }
            macro = ppmacro_new(ftepp_ctx(ftepp), ftepp_tokval(ftepp));
            break;
        default:
            ftepp_error(ftepp, "expected macro name");
            return false;
    }

    (void)ftepp_next(ftepp);

    if (ftepp->token == '(') {
        macro->has_params = true;
        if (!ftepp_define_params(ftepp, macro)) {
            ppmacro_delete(macro);
            return false;
        }
    }

    if (!ftepp_skipspace(ftepp)) {
        ppmacro_delete(macro);
        return false;
    }

    if (!ftepp_define_body(ftepp, macro)) {<commit_msg>Mask out the math constant precedence stuff unless -fftepp-mathdefs<commit_after>/*
 * Copyright (C) 2012, 2013
 *     <NAME>
 *     <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
 * of the Software, and to permit persons to whom the Software is furnished to do
 * so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#include <string.h>
#include <stdlib.h>
#include <sys/stat.h>

#include "gmqcc.h"
#include "lexer.h"

#define HT_MACROS 1024

typedef struct {
    bool on;
    bool was_on;
    bool had_else;
} ppcondition;

typedef struct {
    int   token;
    char *value;
    /* a copy from the lexer */
    union {
        vec3_t v;
        int    i;
        double f;
        int    t; /* type */
    } constval;
} pptoken;

typedef struct {
    lex_ctx_t ctx;

    char   *name;
    char  **params;
    /* yes we need an extra flag since `#define FOO x` is not the same as `#define FOO() x` */
    bool    has_params;
    bool    variadic;

    pptoken **output;
} ppmacro;

typedef struct ftepp_s {
    lex_file    *lex;
    int          token;
    unsigned int errors;

    bool         output_on;
    ppcondition *conditions;
    /*ppmacro    **macros;*/
    ht           macros;  /* hashtable<string, ppmacro*> */
    char        *output_string;

    char        *itemname;
    char        *includename;
    bool         in_macro;

    uint32_t predef_countval;
    uint32_t predef_randval;
} ftepp_t;

/* __DATE__ */
static char *ftepp_predef_date(ftepp_t *context) {
    const struct tm *itime = NULL;
    char            *value = (char*)mem_a(82);
    time_t           rtime;

    (void)context;

    time (&rtime);
    itime = util_localtime(&rtime);
    strftime(value, 82, "\"%b %d %Y\"", itime);

    return value;
}

/* __TIME__ */
static char *ftepp_predef_time(ftepp_t *context) {
    const struct tm *itime = NULL;
    char            *value = (char*)mem_a(82);
    time_t           rtime;

    (void)context;

    time (&rtime);
    itime = util_localtime(&rtime);
    strftime(value, 82, "\"%X\"", itime);

    return value;
}

/* __LINE__ */
static char *ftepp_predef_line(ftepp_t *context) {
    char *value;

    util_asprintf(&value, "%d", (int)context->lex->line);
    return value;
}
/* __FILE__ */
static char *ftepp_predef_file(ftepp_t *context) {
    size_t length = strlen(context->lex->name) + 3; /* two quotes and a terminator */
    char  *value  = (char*)mem_a(length);

    util_snprintf(value, length, "\"%s\"", context->lex->name);
    return value;
}
/* __COUNTER_LAST__ */
static char *ftepp_predef_counterlast(ftepp_t *context) {
    char *value;
    util_asprintf(&value, "%u", context->predef_countval);
    return value;
}
/* __COUNTER__ */
static char *ftepp_predef_counter(ftepp_t *context) {
    char *value;

    context->predef_countval ++;
    util_asprintf(&value, "%u", context->predef_countval);

    return value;
}
/* __RANDOM__ */
static char *ftepp_predef_random(ftepp_t *context) {
    char *value;

    context->predef_randval = (util_rand() % 0xFF) + 1;
    util_asprintf(&value, "%u", context->predef_randval);
    return value;
}
/* __RANDOM_LAST__ */
static char *ftepp_predef_randomlast(ftepp_t *context) {
    char *value;

    util_asprintf(&value, "%u", context->predef_randval);
    return value;
}
/* __TIMESTAMP__ */
static char *ftepp_predef_timestamp(ftepp_t *context) {
    struct stat finfo;
    const char *find;
    char       *value;
    size_t      size;

    if (stat(context->lex->name, &finfo))
        return util_strdup("\"<failed to determine timestamp>\"");

    find = util_ctime(&finfo.st_mtime);
    value = (char*)mem_a(strlen(find) + 1);
    memcpy(&value[1], find, (size = strlen(find)) - 1);

    value[0]    = '"';
    value[size] = '"';

    return value;
}

typedef struct {
    const char   *name;
    char       *(*func)(ftepp_t *);
} ftepp_predef_t;

static const ftepp_predef_t ftepp_predefs[] = {
    { "__LINE__",         &ftepp_predef_line        },
    { "__FILE__",         &ftepp_predef_file        },
    { "__COUNTER__",      &ftepp_predef_counter     },
    { "__COUNTER_LAST__", &ftepp_predef_counterlast },
    { "__RANDOM__",       &ftepp_predef_random      },
    { "__RANDOM_LAST__",  &ftepp_predef_randomlast  },
    { "__DATE__",         &ftepp_predef_date        },
    { "__TIME__",         &ftepp_predef_time        },
    { "__TIME_STAMP__",   &ftepp_predef_timestamp   }
};

static GMQCC_INLINE size_t ftepp_predef_index(const char *name) {
    /* no hashtable here, we simply check for one to exist the naive way */
    size_t i;
    for(i = 1; i < GMQCC_ARRAY_COUNT(ftepp_predefs) + 1; i++)
        if (!strcmp(ftepp_predefs[i-1].name, name))
            return i;
    return 0;
}

bool ftepp_predef_exists(const char *name);
bool ftepp_predef_exists(const char *name) {
    return ftepp_predef_index(name) != 0;
}

/* singleton because we're allowed */
static GMQCC_INLINE char *(*ftepp_predef(const char *name))(ftepp_t *context) {
    size_t i = ftepp_predef_index(name);
    return (i != 0) ? ftepp_predefs[i-1].func : NULL;
}

#define ftepp_tokval(f) ((f)->lex->tok.value)
#define ftepp_ctx(f)    ((f)->lex->tok.ctx)

static void ftepp_errorat(ftepp_t *ftepp, lex_ctx_t ctx, const char *fmt, ...)
{
    va_list ap;

    ftepp->errors++;

    va_start(ap, fmt);
    con_cvprintmsg(ctx, LVL_ERROR, "error", fmt, ap);
    va_end(ap);
}

static void ftepp_error(ftepp_t *ftepp, const char *fmt, ...)
{
    va_list ap;

    ftepp->errors++;

    va_start(ap, fmt);
    con_cvprintmsg(ftepp->lex->tok.ctx, LVL_ERROR, "error", fmt, ap);
    va_end(ap);
}

static bool GMQCC_WARN ftepp_warn(ftepp_t *ftepp, int warntype, const char *fmt, ...)
{
    bool    r;
    va_list ap;

    va_start(ap, fmt);
    r = vcompile_warning(ftepp->lex->tok.ctx, warntype, fmt, ap);
    va_end(ap);
    return r;
}

static pptoken *pptoken_make(ftepp_t *ftepp)
{
    pptoken *token = (pptoken*)mem_a(sizeof(pptoken));
    token->token = ftepp->token;
#if 0
    if (token->token == TOKEN_WHITE)
        token->value = util_strdup(" ");
    else
#else
        token->value = util_strdup(ftepp_tokval(ftepp));
#endif
    memcpy(&token->constval, &ftepp->lex->tok.constval, sizeof(token->constval));
    return token;
}

static GMQCC_INLINE void pptoken_delete(pptoken *self)
{
    mem_d(self->value);
    mem_d(self);
}

static ppmacro *ppmacro_new(lex_ctx_t ctx, const char *name)
{
    ppmacro *macro = (ppmacro*)mem_a(sizeof(ppmacro));

    (void)ctx;
    memset(macro, 0, sizeof(*macro));
    macro->name = util_strdup(name);
    return macro;
}

static void ppmacro_delete(ppmacro *self)
{
    size_t i;
    for (i = 0; i < vec_size(self->params); ++i)
        mem_d(self->params[i]);
    vec_free(self->params);
    for (i = 0; i < vec_size(self->output); ++i)
        pptoken_delete(self->output[i]);
    vec_free(self->output);
    mem_d(self->name);
    mem_d(self);
}

static ftepp_t* ftepp_new(void)
{
    ftepp_t *ftepp;

    ftepp = (ftepp_t*)mem_a(sizeof(*ftepp));
    memset(ftepp, 0, sizeof(*ftepp));

    ftepp->macros          = util_htnew(HT_MACROS);
    ftepp->output_on       = true;
    ftepp->predef_countval = 0;
    ftepp->predef_randval  = 0;

    return ftepp;
}

static GMQCC_INLINE void ftepp_flush_do(ftepp_t *self)
{
    vec_free(self->output_string);
}

static void ftepp_delete(ftepp_t *self)
{
    ftepp_flush_do(self);
    if (self->itemname)
        mem_d(self->itemname);
    if (self->includename)
        vec_free(self->includename);

    util_htrem(self->macros, (void (*)(void*))&ppmacro_delete);

    vec_free(self->conditions);
    if (self->lex)
        lex_close(self->lex);
    mem_d(self);
}

static void ftepp_out(ftepp_t *ftepp, const char *str, bool ignore_cond)
{
    if (ignore_cond || ftepp->output_on)
    {
        size_t len;
        char  *data;
        len = strlen(str);
        data = vec_add(ftepp->output_string, len);
        memcpy(data, str, len);
    }
}

static GMQCC_INLINE void ftepp_update_output_condition(ftepp_t *ftepp)
{
    size_t i;
    ftepp->output_on = true;
    for (i = 0; i < vec_size(ftepp->conditions); ++i)
        ftepp->output_on = ftepp->output_on && ftepp->conditions[i].on;
}

static GMQCC_INLINE ppmacro* ftepp_macro_find(ftepp_t *ftepp, const char *name)
{
    return (ppmacro*)util_htget(ftepp->macros, name);
}

static GMQCC_INLINE void ftepp_macro_delete(ftepp_t *ftepp, const char *name)
{
    util_htrm(ftepp->macros, name, (void (*)(void*))&ppmacro_delete);
}

static GMQCC_INLINE int ftepp_next(ftepp_t *ftepp)
{
    return (ftepp->token = lex_do(ftepp->lex));
}

/* Important: this does not skip newlines! */
static bool ftepp_skipspace(ftepp_t *ftepp)
{
    if (ftepp->token != TOKEN_WHITE)
        return true;
    while (ftepp_next(ftepp) == TOKEN_WHITE) {}
    if (ftepp->token >= TOKEN_EOF) {
        ftepp_error(ftepp, "unexpected end of preprocessor directive");
        return false;
    }
    return true;
}

/* this one skips EOLs as well */
static bool ftepp_skipallwhite(ftepp_t *ftepp)
{
    if (ftepp->token != TOKEN_WHITE && ftepp->token != TOKEN_EOL)
        return true;
    do {
        ftepp_next(ftepp);
    } while (ftepp->token == TOKEN_WHITE || ftepp->token == TOKEN_EOL);
    if (ftepp->token >= TOKEN_EOF) {
        ftepp_error(ftepp, "unexpected end of preprocessor directive");
        return false;
    }
    return true;
}

/**
 * The huge macro parsing code...
 */
static bool ftepp_define_params(ftepp_t *ftepp, ppmacro *macro)
{
    do {
        ftepp_next(ftepp);
        if (!ftepp_skipspace(ftepp))
            return false;
        if (ftepp->token == ')')
            break;
        switch (ftepp->token) {
            case TOKEN_IDENT:
            case TOKEN_TYPENAME:
            case TOKEN_KEYWORD:
                vec_push(macro->params, util_strdup(ftepp_tokval(ftepp)));
                break;
            case TOKEN_DOTS:
                macro->variadic = true;
                break;
            default:
                ftepp_error(ftepp, "unexpected token in parameter list");
                return false;
        }
        ftepp_next(ftepp);
        if (!ftepp_skipspace(ftepp))
            return false;
        if (macro->variadic && ftepp->token != ')') {
            ftepp_error(ftepp, "cannot have parameters after the variadic parameters");
            return false;
        }
    } while (ftepp->token == ',');

    if (ftepp->token != ')') {
        ftepp_error(ftepp, "expected closing paren after macro parameter list");
        return false;
    }
    ftepp_next(ftepp);
    /* skipspace happens in ftepp_define */
    return true;
}

static bool ftepp_define_body(ftepp_t *ftepp, ppmacro *macro)
{
    pptoken *ptok;
    while (ftepp->token != TOKEN_EOL && ftepp->token < TOKEN_EOF) {
        bool   subscript = false;
        size_t index     = 0;
        if (macro->variadic && !strcmp(ftepp_tokval(ftepp), "__VA_ARGS__")) {
            subscript = !!(ftepp_next(ftepp) == '#');

            if (subscript && ftepp_next(ftepp) != '#') {
                ftepp_error(ftepp, "expected `##` in __VA_ARGS__ for subscripting");
                return false;
            } else if (subscript) {
                if (ftepp_next(ftepp) == '[') {
                    if (ftepp_next(ftepp) != TOKEN_INTCONST) {
                        ftepp_error(ftepp, "expected index for __VA_ARGS__ subscript");
                        return false;
                    }

                    index = (int)strtol(ftepp_tokval(ftepp), NULL, 10);

                    if (ftepp_next(ftepp) != ']') {
                        ftepp_error(ftepp, "expected `]` in __VA_ARGS__ subscript");
                        return false;
                    }

                    /*
                     * mark it as an array to be handled later as such and not
                     * as traditional __VA_ARGS__
                     */
                    ftepp->token = TOKEN_VA_ARGS_ARRAY;
                    ptok = pptoken_make(ftepp);
                    ptok->constval.i = index;
                    vec_push(macro->output, ptok);
                    ftepp_next(ftepp);
                } else {
                    ftepp_error(ftepp, "expected `[` for subscripting of __VA_ARGS__");
                    return false;
                }
            } else {
                int old = ftepp->token;
                ftepp->token = TOKEN_VA_ARGS;
                ptok = pptoken_make(ftepp);
                vec_push(macro->output, ptok);
                ftepp->token = old;
            }
        }
        else if (macro->variadic && !strcmp(ftepp_tokval(ftepp), "__VA_COUNT__")) {
            ftepp->token = TOKEN_VA_COUNT;
            ptok         = pptoken_make(ftepp);
            vec_push(macro->output, ptok);
            ftepp_next(ftepp);
        } else {
            ptok = pptoken_make(ftepp);
            vec_push(macro->output, ptok);
            ftepp_next(ftepp);
        }
    }
    /* recursive expansion can cause EOFs here */
    if (ftepp->token != TOKEN_EOL && ftepp->token != TOKEN_EOF) {
        ftepp_error(ftepp, "unexpected junk after macro or unexpected end of file");
        return false;
    }
    return true;
}

static const char *ftepp_math_constants[][2] = {
    { "M_E",        "2.7182818284590452354"  }, /* e          */
    { "M_LOG2E",    "1.4426950408889634074"  }, /* log_2 e    */
    { "M_LOG10E",   "0.43429448190325182765" }, /* log_10 e   */
    { "M_LN2",      "0.69314718055994530942" }, /* log_e 2    */
    { "M_LN10",     "2.30258509299404568402" }, /* log_e 10   */
    { "M_PI",       "3.14159265358979323846" }, /* pi         */
    { "M_PI_2",     "1.57079632679489661923" }, /* pi/2       */
    { "M_PI_4",     "0.78539816339744830962" }, /* pi/4       */
    { "M_1_PI",     "0.31830988618379067154" }, /* 1/pi       */
    { "M_2_PI",     "0.63661977236758134308" }, /* 2/pi       */
    { "M_2_SQRTPI", "1.12837916709551257390" }, /* 2/sqrt(pi) */
    { "M_SQRT2",    "1.41421356237309504880" }, /* sqrt(2)    */
    { "M_SQRT1_2",  "0.70710678118654752440" }, /* 1/sqrt(2)  */
    { "M_TAU",      "6.28318530717958647692" }  /* pi*2       */
};

static bool ftepp_define(ftepp_t *ftepp)
{
    ppmacro *macro = NULL;
    size_t l = ftepp_ctx(ftepp).line;
    size_t i;
    bool   mathconstant = false;

    (void)ftepp_next(ftepp);
    if (!ftepp_skipspace(ftepp))
        return false;

    switch (ftepp->token) {
        case TOKEN_IDENT:
        case TOKEN_TYPENAME:
        case TOKEN_KEYWORD:
            if (OPTS_FLAG(FTEPP_MATHDEFS)) {
                for (i = 0; i < GMQCC_ARRAY_COUNT(ftepp_math_constants); i++) {
                    if (!strcmp(ftepp_math_constants[i][0], ftepp_tokval(ftepp))) {
                        mathconstant = true;
                        break;
                    }
                }
            }

            macro = ftepp_macro_find(ftepp, ftepp_tokval(ftepp));

            if (OPTS_FLAG(FTEPP_MATHDEFS)) {
                /* user defined ones take precedence */
                if (macro && mathconstant) {
                    ftepp_macro_delete(ftepp, ftepp_tokval(ftepp));
                    macro = NULL;
                }
            }

            if (macro && ftepp->output_on) {
                if (ftepp_warn(ftepp, WARN_CPP, "redefining `%s`", ftepp_tokval(ftepp)))
                    return false;
                ftepp_macro_delete(ftepp, ftepp_tokval(ftepp));
            }
            macro = ppmacro_new(ftepp_ctx(ftepp), ftepp_tokval(ftepp));
            break;
        default:
            ftepp_error(ftepp, "expected macro name");
            return false;
    }

    (void)ftepp_next(ftepp);

    if (ftepp->token == '(') {
        macro->has_params = true;
        if (!ftepp_define_params(ftepp, macro)) {
            ppmacro_delete(macro);
            return false;
        }
    }

    if (!ftepp_skipspace(ftepp)) {
        ppmacro_delete(macro);
        return false;
    }

    if (!ftepp_define_body(ftepp, macro)) {
{'fraction_non_alphanumeric': 0.12881679389312978, 'fraction_numerical': 0.0223043893129771, 'mean_word_length': 2.3300565981531127, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 223, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5635075', 'n_tokens_mistral': 12564, 'n_tokens_neox': 11597, 'n_words': 2550}