<commit_before>/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/*  Fluent Bit
 *  ==========
 *  Copyright (C) 2015-2022 The Fluent Bit Authors
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#include <fluent-bit/flb_info.h>
#include <fluent-bit/flb_log.h>
#include <fluent-bit/flb_mem.h>
#include <fluent-bit/flb_str.h>
#include <fluent-bit/flb_parser.h>
#include <fluent-bit/flb_parser_decoder.h>
#include <fluent-bit/flb_time.h>
#include <fluent-bit/flb_error.h>
#include <fluent-bit/flb_utils.h>
#include <fluent-bit/flb_config.h>
#include <fluent-bit/flb_strptime.h>
#include <fluent-bit/flb_env.h>
#include <fluent-bit/flb_str.h>
#include <fluent-bit/flb_kv.h>
#include <fluent-bit/flb_config_format.h>
#include <fluent-bit/multiline/flb_ml.h>
#include <fluent-bit/multiline/flb_ml_parser.h>
#include <fluent-bit/multiline/flb_ml_rule.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <limits.h>
#include <string.h>

static inline uint32_t digits10(uint64_t v) {
    if (v < 10) return 1;
    if (v < 100) return 2;
    if (v < 1000) return 3;
    if (v < 1000000000000UL) {
        if (v < 100000000UL) {
            if (v < 1000000) {
                if (v < 10000) return 4;
                return 5 + (v >= 100000);
            }
            return 7 + (v >= 10000000UL);
        }
        if (v < 10000000000UL) {
            return 9 + (v >= 1000000000UL);
        }
        return 11 + (v >= 100000000000UL);
    }
    return 12 + digits10(v / 1000000000000UL);
}

static unsigned u64_to_str(uint64_t value, char* dst) {
    static const char digits[201] =
        "0001020304050607080910111213141516171819"
        "2021222324252627282930313233343536373839"
        "4041424344454647484950515253545556575859"
        "6061626364656667686970717273747576777879"
        "8081828384858687888990919293949596979899";
    uint32_t const length = digits10(value);
    uint32_t next = length - 1;
    while (value >= 100) {
        int const i = (value % 100) * 2;
        value /= 100;
        dst[next] = digits[i + 1];
        dst[next - 1] = digits[i];
        next -= 2;
    }

    /* Handle last 1-2 digits */
    if (value < 10) {
        dst[next] = '0' + (uint32_t) value;
    } else {
        int i = (uint32_t) value * 2;
        dst[next] = digits[i + 1];
        dst[next - 1] = digits[i];
    }
    return length;
}

int flb_parser_regex_do(struct flb_parser *parser,
                        const char *buf, size_t length,
                        void **out_buf, size_t *out_size,
                        struct flb_time *out_time);

int flb_parser_json_do(struct flb_parser *parser,
                       const char *buf, size_t length,
                       void **out_buf, size_t *out_size,
                       struct flb_time *out_time);

int flb_parser_ltsv_do(struct flb_parser *parser,
                       const char *buf, size_t length,
                       void **out_buf, size_t *out_size,
                       struct flb_time *out_time);

int flb_parser_logfmt_do(struct flb_parser *parser,
                         const char *buf, size_t length,
                         void **out_buf, size_t *out_size,
                         struct flb_time *out_time);

/*
 * This function is used to free all aspects of a parser
 * which is provided by the caller of flb_create_parser.
 * Specifically, this function frees all but parser.types and
 * parser.decoders from a parser.
 *
 * This function is only to be used in parser creation routines.
 */
static void flb_interim_parser_destroy(struct flb_parser *parser)
{
    if (parser->type == FLB_PARSER_REGEX) {
        flb_regex_destroy(parser->regex);
        flb_free(parser->p_regex);
    }

    flb_free(parser->name);
    if (parser->time_fmt) {
        flb_free(parser->time_fmt);
        flb_free(parser->time_fmt_full);
    }
    if (parser->time_fmt_year) {
        flb_free(parser->time_fmt_year);
    }
    if (parser->time_key) {
        flb_free(parser->time_key);
    }

    mk_list_del(&parser->_head);
    flb_free(parser);
}

struct flb_parser *flb_parser_create(const char *name, const char *format,
                                     const char *p_regex,
                                     int skip_empty,
                                     const char *time_fmt, const char *time_key,
                                     const char *time_offset,
                                     int time_keep,
                                     int time_strict,
                                     struct flb_parser_types *types,
                                     int types_len,
                                     struct mk_list *decoders,
                                     struct flb_config *config)
{
    int ret;
    int len;
    int diff = 0;
    int size;
    int is_epoch = FLB_FALSE;
    char *tmp;
    char *timeptr;
    struct mk_list *head;
    struct flb_parser *p;
    struct flb_regex *regex;

    /* Iterate current parsers and make sure the new one don't exists */
    mk_list_foreach(head, &config->parsers) {
        p = mk_list_entry(head, struct flb_parser, _head);
        if (p->name && strcmp(p->name, name) == 0) {
            flb_error("[parser] parser named '%s' already exists, skip.",
                      name);
            return NULL;
        }
    }

    /* Allocate context */
    p = flb_calloc(1, sizeof(struct flb_parser));
    if (!p) {
        flb_errno();
        return NULL;
    }
    p->decoders = decoders;
    mk_list_add(&p->_head, &config->parsers);

    /* Format lookup */
    if (strcasecmp(format, "regex") == 0) {
        p->type = FLB_PARSER_REGEX;
    }
    else if (strcasecmp(format, "json") == 0) {
        p->type = FLB_PARSER_JSON;
    }
    else if (strcasecmp(format, "ltsv") == 0) {
        p->type = FLB_PARSER_LTSV;
    }
    else if (strcasecmp(format, "logfmt") == 0) {
        p->type = FLB_PARSER_LOGFMT;
    }
    else {
        flb_error("[parser:%s] Invalid format %s", name, format);
        mk_list_del(&p->_head);
        flb_free(p);
        return NULL;
    }

    if (p->type == FLB_PARSER_REGEX) {
        if (!p_regex) {
            flb_error("[parser:%s] Invalid regex pattern", name);
            mk_list_del(&p->_head);
            flb_free(p);
            return NULL;
        }

        regex = flb_regex_create(p_regex);
        if (!regex) {
            flb_error("[parser:%s] Invalid regex pattern %s", name, p_regex);
            mk_list_del(&p->_head);
            flb_free(p);
            return NULL;
        }
        p->regex = regex;
        p->skip_empty = skip_empty;
        p->p_regex = flb_strdup(p_regex);
    }

    p->name = flb_strdup(name);

    if (time_fmt) {
        p->time_fmt_full = flb_strdup(time_fmt);
        p->time_fmt = flb_strdup(time_fmt);

        /* Check if the format is considering the year */
        if (strstr(p->time_fmt, "%Y") || strstr(p->time_fmt, "%y")) {
            p->time_with_year = FLB_TRUE;
        }
        else if (strstr(p->time_fmt, "%s")) {
            is_epoch = FLB_TRUE;
            p->time_with_year = FLB_TRUE;
        }
        else {
            size = strlen(p->time_fmt);
            p->time_with_year = FLB_FALSE;
            p->time_fmt_year = flb_malloc(size + 4);
            if (!p->time_fmt_year) {
                flb_errno();
                flb_interim_parser_destroy(p);
                return NULL;
            }

            /* Append the year at the beginning */
            tmp = p->time_fmt_year;
            *tmp++ = '%';
            *tmp++ = 'Y';
            *tmp++ = ' ';

            memcpy(tmp, p->time_fmt, size);
            tmp += size;
            *tmp++ = '\0';
        }

        /* Check if the format contains a timezone (%z) */
        if (strstr(p->time_fmt, "%z") || strstr(p->time_fmt, "%Z") ||
            strstr(p->time_fmt, "%SZ") || strstr(p->time_fmt, "%S.%LZ")) {
#if defined(FLB_HAVE_GMTOFF) || !defined(FLB_HAVE_SYSTEM_STRPTIME)
            p->time_with_tz = FLB_TRUE;
#else
            flb_error("[parser] timezone offset not supported");
            flb_error("[parser] you cannot use %%z/%%Z on this platform");
            flb_interim_parser_destroy(p);
            return NULL;
#endif
        }

        /*
         * Check if the format expect fractional seconds
         *
         * Since strptime(3) does not support fractional seconds, this
         * requires a workaround/hack in our parser. This is a known
         * issue and addressed in different ways in other languages.
         *
         * The following links are a good reference:
         *
         * - http://stackoverflow.com/questions/7114690/how-to-parse-syslog-timestamp
         * - http://code.activestate.com/lists/python-list/521885
         */
        if (is_epoch == FLB_TRUE || p->time_with_year == FLB_TRUE) {
            timeptr = p->time_fmt;
        }
        else {
            timeptr = p->time_fmt_year;
        }

        tmp = strstr(timeptr, "%L");
        if (tmp) {
            tmp[0] = '\0';
            tmp[1] = '\0';
            p->time_frac_secs = (tmp + 2);
        }

        /* Optional fixed timezone offset */
        if (time_offset) {
            diff = 0;
            len = strlen(time_offset);
            ret = flb_parser_tzone_offset(time_offset, len, &diff);
            if (ret == -1) {
                flb_interim_parser_destroy(p);
                return NULL;
            }
            p->time_offset = diff;
        }
    }

    if (time_key) {
        p->time_key = flb_strdup(time_key);
    }

    p->time_keep = time_keep;
    p->time_strict = time_strict;
    p->types = types;
    p->types_len = types_len;
    return p;
}

void flb_parser_destroy(struct flb_parser *parser)
{
    int i = 0;

    if (parser->type == FLB_PARSER_REGEX) {
        flb_regex_destroy(parser->regex);
        flb_free(parser->p_regex);
    }

    flb_free(parser->name);
    if (parser->time_fmt) {
        flb_free(parser->time_fmt);
        flb_free(parser->time_fmt_full);
    }
    if (parser->time_fmt_year) {
        flb_free(parser->time_fmt_year);
    }
    if (parser->time_key) {
        flb_free(parser->time_key);
    }
    if (parser->types_len != 0) {
        for (i=0; i<parser->types_len; i++){
            flb_free(parser->types[i].key);
        }
        flb_free(parser->types);
    }

    if (parser->decoders) {
        flb_parser_decoder_list_destroy(parser->decoders);
    }

    mk_list_del(&parser->_head);
    flb_free(parser);
}

void flb_parser_exit(struct flb_config *config)
{
    struct mk_list *tmp;
    struct mk_list *head;
    struct flb_parser *parser;

    /* release 'parsers' */
    mk_list_foreach_safe(head, tmp, &config->parsers) {
        parser = mk_list_entry(head, struct flb_parser, _head);
        flb_parser_destroy(parser);
    }

    /* release 'multiline parsers' */
    flb_ml_exit(config);
}

static int proc_types_str(const char *types_str, struct flb_parser_types **types)
{
    int i = 0;
    int types_num = 0;
    char *type_str = NULL;
    size_t len;
    struct mk_list *split;
    struct mk_list *head;
    struct flb_split_entry *sentry;

    split = flb_utils_split(types_str, ' ', 256);
    types_num = mk_list_size(split);
    *types = flb_malloc(sizeof(struct flb_parser_types) * types_num);

    for(i=0; i<types_num; i++){
        (*types)[i].key = NULL;
        (*types)[i].type = FLB_PARSER_TYPE_STRING;
    }
    i = 0;
    mk_list_foreach(head ,split) {
        sentry = mk_list_entry(head, struct flb_split_entry ,_head);
        type_str = strchr(sentry->value ,':');

        if (type_str == NULL) {
            i++;
            continue;
        }
        len = type_str - sentry->value;
        (*types)[i].key = flb_strndup(sentry->value, len);
        (*types)[i].key_len = len;

        type_str++;
        if (!strcasecmp(type_str, "integer")) {
            (*types)[i].type = FLB_PARSER_TYPE_INT;
        }
        else if(!strcasecmp(type_str, "bool")) {
            (*types)[i].type = FLB_PARSER_TYPE_BOOL;
        }
        else if(!strcasecmp(type_str, "float")){
            (*types)[i].type = FLB_PARSER_TYPE_FLOAT;
        }
        else if(!strcasecmp(type_str, "hex")){
            (*types)[i].type = FLB_PARSER_TYPE_HEX;
        }
        else {
            (*types)[i].type = FLB_PARSER_TYPE_STRING;
        }
        i++;
    }
    flb_utils_split_free(split);

    return i;
}

static flb_sds_t get_parser_key(struct flb_config *config,
                                struct flb_cf *cf, struct flb_cf_section *s,
                                char *key)

{
    char *tmp;
    flb_sds_t val;

    tmp = flb_cf_section_property_get(cf, s, key);
    if (!tmp) {
        return NULL;
    }

    val = flb_env_var_translate(config->env, tmp);
    if (!val) {
        return NULL;
    }

    if (flb_sds_len(val) == 0) {
        flb_sds_destroy(val);
        return NULL;
    }

    return val;
}

/* Config file: read 'parser' definitions */
static int parser_conf_file(const char *cfg, struct flb_cf *cf,
                            struct flb_config *config)
{
    flb_sds_t name;
    flb_sds_t format;
    flb_sds_t regex;
    flb_sds_t time_fmt;
    flb_sds_t time_key;
    flb_sds_t time_offset;
    flb_sds_t types_str;
    flb_sds_t tmp_str;
    int skip_empty;
    int time_keep;
    int time_strict;
    int types_len;
    struct mk_list *head;
    struct mk_list *decoders = NULL;
    struct flb_cf_section *s;
    struct flb_parser_types *types = NULL;

    /* Read all 'parser' sections */
    mk_list_foreach(head, &cf->parsers) {
        name = NULL;
        format = NULL;
        regex = NULL;
        time_fmt = NULL;
        time_key = NULL;
        time_offset = NULL;
        types_str = NULL;
        tmp_str = NULL;

        /* retrieve the section context */
        s = mk_list_entry(head, struct flb_cf_section, _head_section);

        /* name */
        name = get_parser_key(config, cf, s, "name");
        if (!name) {
            flb_error("[parser] no parser 'name' found in file '%s'", cfg);
            goto fconf_error;
        }

        /* format */
        format = get_parser_key(config, cf, s, "format");
        if (!format) {
            flb_error("[parser] no parser 'format' found for '%s' in file '%s'",
                      name, cfg);
            goto fconf_error;
        }

        /* regex (if 'format' == 'regex') */
        regex = get_parser_key(config, cf, s, "regex");
        if (!regex && strcmp(format, "regex") == 0) {
            flb_error("[parser] no parser 'regex' found for '%s' in file '%s",
                      name, cfg);
            goto fconf_error;
        }
        
        /* skip_empty_values */
        skip_empty = FLB_TRUE;
        tmp_str = get_parser_key(config, cf, s, "skip_empty_values");
        if (tmp_str) {
            skip_empty = flb_utils_bool(tmp_str);
            flb_sds_destroy(tmp_str);
        }

        /* time_format */
        time_fmt = get_parser_key(config, cf, s, "time_format");

        /* time_key */
        time_key = get_parser_key(config, cf, s, "time_key");

        /* time_keep */
        time_keep = FLB_FALSE;
        tmp_str = get_parser_key(config, cf, s, "time_keep");
        if (tmp_str) {
            time_keep = flb_utils_bool(tmp_str);
            flb_sds_destroy(tmp_str);
        }

        /* time_strict */
        time_strict = FLB_TRUE;
        tmp_str = get_parser_key(config, cf, s, "time_strict");
        if (tmp_str) {
            time_strict = flb_utils_bool(tmp_str);
            flb_sds_destroy(tmp_str);
        }

        /* time_offset (UTC offset) */
        time_offset = get_parser_key(config, cf, s, "time_offset");

        /* types */
        types_str = get_parser_key(config, cf, s, "types");
        if (types_str) {
            types_len = proc_types_str(types_str, &types);
        }
        else {
            types_len = 0;
        }

        /* Decoders */
        decoders = flb_parser_decoder_list_create(s);

        /* Create the parser context */
        if (!flb_parser_create(name, format, regex, skip_empty,
                               time_fmt, time_key, time_offset, time_keep, time_strict,
                               types, types_len, decoders, config)) {
            goto fconf_error;
        }

        flb_debug("[parser] new parser registered: %s", name);

        flb_sds_destroy(name);
        flb_sds_destroy(format);

        if (regex) {
            flb_sds_destroy(regex);
        }
        if (time_fmt) {
            flb_sds_destroy(time_fmt);
        }
        if (time_key) {
            flb_sds_destroy(time_key);
        }
        if (time_offset) {
            flb_sds_destroy(time_offset);
        }
        if (types_str) {
            flb_sds_destroy(types_str);
        }
        decoders = NULL;
    }

    return 0;

 fconf_error:
    flb_sds_destroy(name);
    flb_sds_destroy(format);
    if (regex) {
        flb_sds_destroy(regex);
    }
    if (time_fmt) {
        flb_sds_destroy(time_fmt);
    }
    if (time_key) {
        flb_sds_destroy(time_key);
    }
    if (time_offset) {
        flb_sds_destroy(time_offset);
    }
    if (types_str) {
        flb_sds_destroy(types_str);
    }
    if (decoders) {
        flb_parser_decoder_list_destroy(decoders);
    }
    return -1;
}

static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser,
                                      struct flb_cf_section *section,
                                      struct flb_config *config)
{
    int ret;
    char *to_state = NULL;
    struct mk_list list;
    struct mk_list *head;
    struct flb_kv *entry;
    struct flb_slist_entry *from_state;
    struct flb_slist_entry *regex_pattern;
    struct flb_slist_entry *tmp;

    mk_list_foreach(head, &section->properties) {
        entry = mk_list_entry(head, struct flb_kv, _head);

        /* only process 'rule' keys */
        if (strcasecmp(entry->key, "rule") != 0) {<commit_msg>parser: free allocated types<commit_after>/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/*  Fluent Bit
 *  ==========
 *  Copyright (C) 2015-2022 The Fluent Bit Authors
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#include <fluent-bit/flb_info.h>
#include <fluent-bit/flb_log.h>
#include <fluent-bit/flb_mem.h>
#include <fluent-bit/flb_str.h>
#include <fluent-bit/flb_parser.h>
#include <fluent-bit/flb_parser_decoder.h>
#include <fluent-bit/flb_time.h>
#include <fluent-bit/flb_error.h>
#include <fluent-bit/flb_utils.h>
#include <fluent-bit/flb_config.h>
#include <fluent-bit/flb_strptime.h>
#include <fluent-bit/flb_env.h>
#include <fluent-bit/flb_str.h>
#include <fluent-bit/flb_kv.h>
#include <fluent-bit/flb_config_format.h>
#include <fluent-bit/multiline/flb_ml.h>
#include <fluent-bit/multiline/flb_ml_parser.h>
#include <fluent-bit/multiline/flb_ml_rule.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <limits.h>
#include <string.h>

static inline uint32_t digits10(uint64_t v) {
    if (v < 10) return 1;
    if (v < 100) return 2;
    if (v < 1000) return 3;
    if (v < 1000000000000UL) {
        if (v < 100000000UL) {
            if (v < 1000000) {
                if (v < 10000) return 4;
                return 5 + (v >= 100000);
            }
            return 7 + (v >= 10000000UL);
        }
        if (v < 10000000000UL) {
            return 9 + (v >= 1000000000UL);
        }
        return 11 + (v >= 100000000000UL);
    }
    return 12 + digits10(v / 1000000000000UL);
}

static unsigned u64_to_str(uint64_t value, char* dst) {
    static const char digits[201] =
        "0001020304050607080910111213141516171819"
        "2021222324252627282930313233343536373839"
        "4041424344454647484950515253545556575859"
        "6061626364656667686970717273747576777879"
        "8081828384858687888990919293949596979899";
    uint32_t const length = digits10(value);
    uint32_t next = length - 1;
    while (value >= 100) {
        int const i = (value % 100) * 2;
        value /= 100;
        dst[next] = digits[i + 1];
        dst[next - 1] = digits[i];
        next -= 2;
    }

    /* Handle last 1-2 digits */
    if (value < 10) {
        dst[next] = '0' + (uint32_t) value;
    } else {
        int i = (uint32_t) value * 2;
        dst[next] = digits[i + 1];
        dst[next - 1] = digits[i];
    }
    return length;
}

int flb_parser_regex_do(struct flb_parser *parser,
                        const char *buf, size_t length,
                        void **out_buf, size_t *out_size,
                        struct flb_time *out_time);

int flb_parser_json_do(struct flb_parser *parser,
                       const char *buf, size_t length,
                       void **out_buf, size_t *out_size,
                       struct flb_time *out_time);

int flb_parser_ltsv_do(struct flb_parser *parser,
                       const char *buf, size_t length,
                       void **out_buf, size_t *out_size,
                       struct flb_time *out_time);

int flb_parser_logfmt_do(struct flb_parser *parser,
                         const char *buf, size_t length,
                         void **out_buf, size_t *out_size,
                         struct flb_time *out_time);

/*
 * This function is used to free all aspects of a parser
 * which is provided by the caller of flb_create_parser.
 * Specifically, this function frees all but parser.types and
 * parser.decoders from a parser.
 *
 * This function is only to be used in parser creation routines.
 */
static void flb_interim_parser_destroy(struct flb_parser *parser)
{
    if (parser->type == FLB_PARSER_REGEX) {
        flb_regex_destroy(parser->regex);
        flb_free(parser->p_regex);
    }

    flb_free(parser->name);
    if (parser->time_fmt) {
        flb_free(parser->time_fmt);
        flb_free(parser->time_fmt_full);
    }
    if (parser->time_fmt_year) {
        flb_free(parser->time_fmt_year);
    }
    if (parser->time_key) {
        flb_free(parser->time_key);
    }

    mk_list_del(&parser->_head);
    flb_free(parser);
}

struct flb_parser *flb_parser_create(const char *name, const char *format,
                                     const char *p_regex,
                                     int skip_empty,
                                     const char *time_fmt, const char *time_key,
                                     const char *time_offset,
                                     int time_keep,
                                     int time_strict,
                                     struct flb_parser_types *types,
                                     int types_len,
                                     struct mk_list *decoders,
                                     struct flb_config *config)
{
    int ret;
    int len;
    int diff = 0;
    int size;
    int is_epoch = FLB_FALSE;
    char *tmp;
    char *timeptr;
    struct mk_list *head;
    struct flb_parser *p;
    struct flb_regex *regex;

    /* Iterate current parsers and make sure the new one don't exists */
    mk_list_foreach(head, &config->parsers) {
        p = mk_list_entry(head, struct flb_parser, _head);
        if (p->name && strcmp(p->name, name) == 0) {
            flb_error("[parser] parser named '%s' already exists, skip.",
                      name);
            return NULL;
        }
    }

    /* Allocate context */
    p = flb_calloc(1, sizeof(struct flb_parser));
    if (!p) {
        flb_errno();
        return NULL;
    }
    p->decoders = decoders;
    mk_list_add(&p->_head, &config->parsers);

    /* Format lookup */
    if (strcasecmp(format, "regex") == 0) {
        p->type = FLB_PARSER_REGEX;
    }
    else if (strcasecmp(format, "json") == 0) {
        p->type = FLB_PARSER_JSON;
    }
    else if (strcasecmp(format, "ltsv") == 0) {
        p->type = FLB_PARSER_LTSV;
    }
    else if (strcasecmp(format, "logfmt") == 0) {
        p->type = FLB_PARSER_LOGFMT;
    }
    else {
        flb_error("[parser:%s] Invalid format %s", name, format);
        mk_list_del(&p->_head);
        flb_free(p);
        return NULL;
    }

    if (p->type == FLB_PARSER_REGEX) {
        if (!p_regex) {
            flb_error("[parser:%s] Invalid regex pattern", name);
            mk_list_del(&p->_head);
            flb_free(p);
            return NULL;
        }

        regex = flb_regex_create(p_regex);
        if (!regex) {
            flb_error("[parser:%s] Invalid regex pattern %s", name, p_regex);
            mk_list_del(&p->_head);
            flb_free(p);
            return NULL;
        }
        p->regex = regex;
        p->skip_empty = skip_empty;
        p->p_regex = flb_strdup(p_regex);
    }

    p->name = flb_strdup(name);

    if (time_fmt) {
        p->time_fmt_full = flb_strdup(time_fmt);
        p->time_fmt = flb_strdup(time_fmt);

        /* Check if the format is considering the year */
        if (strstr(p->time_fmt, "%Y") || strstr(p->time_fmt, "%y")) {
            p->time_with_year = FLB_TRUE;
        }
        else if (strstr(p->time_fmt, "%s")) {
            is_epoch = FLB_TRUE;
            p->time_with_year = FLB_TRUE;
        }
        else {
            size = strlen(p->time_fmt);
            p->time_with_year = FLB_FALSE;
            p->time_fmt_year = flb_malloc(size + 4);
            if (!p->time_fmt_year) {
                flb_errno();
                flb_interim_parser_destroy(p);
                return NULL;
            }

            /* Append the year at the beginning */
            tmp = p->time_fmt_year;
            *tmp++ = '%';
            *tmp++ = 'Y';
            *tmp++ = ' ';

            memcpy(tmp, p->time_fmt, size);
            tmp += size;
            *tmp++ = '\0';
        }

        /* Check if the format contains a timezone (%z) */
        if (strstr(p->time_fmt, "%z") || strstr(p->time_fmt, "%Z") ||
            strstr(p->time_fmt, "%SZ") || strstr(p->time_fmt, "%S.%LZ")) {
#if defined(FLB_HAVE_GMTOFF) || !defined(FLB_HAVE_SYSTEM_STRPTIME)
            p->time_with_tz = FLB_TRUE;
#else
            flb_error("[parser] timezone offset not supported");
            flb_error("[parser] you cannot use %%z/%%Z on this platform");
            flb_interim_parser_destroy(p);
            return NULL;
#endif
        }

        /*
         * Check if the format expect fractional seconds
         *
         * Since strptime(3) does not support fractional seconds, this
         * requires a workaround/hack in our parser. This is a known
         * issue and addressed in different ways in other languages.
         *
         * The following links are a good reference:
         *
         * - http://stackoverflow.com/questions/7114690/how-to-parse-syslog-timestamp
         * - http://code.activestate.com/lists/python-list/521885
         */
        if (is_epoch == FLB_TRUE || p->time_with_year == FLB_TRUE) {
            timeptr = p->time_fmt;
        }
        else {
            timeptr = p->time_fmt_year;
        }

        tmp = strstr(timeptr, "%L");
        if (tmp) {
            tmp[0] = '\0';
            tmp[1] = '\0';
            p->time_frac_secs = (tmp + 2);
        }

        /* Optional fixed timezone offset */
        if (time_offset) {
            diff = 0;
            len = strlen(time_offset);
            ret = flb_parser_tzone_offset(time_offset, len, &diff);
            if (ret == -1) {
                flb_interim_parser_destroy(p);
                return NULL;
            }
            p->time_offset = diff;
        }
    }

    if (time_key) {
        p->time_key = flb_strdup(time_key);
    }

    p->time_keep = time_keep;
    p->time_strict = time_strict;
    p->types = types;
    p->types_len = types_len;
    return p;
}

void flb_parser_destroy(struct flb_parser *parser)
{
    int i = 0;

    if (parser->type == FLB_PARSER_REGEX) {
        flb_regex_destroy(parser->regex);
        flb_free(parser->p_regex);
    }

    flb_free(parser->name);
    if (parser->time_fmt) {
        flb_free(parser->time_fmt);
        flb_free(parser->time_fmt_full);
    }
    if (parser->time_fmt_year) {
        flb_free(parser->time_fmt_year);
    }
    if (parser->time_key) {
        flb_free(parser->time_key);
    }
    if (parser->types_len != 0) {
        for (i=0; i<parser->types_len; i++){
            flb_free(parser->types[i].key);
        }
        flb_free(parser->types);
    }

    if (parser->decoders) {
        flb_parser_decoder_list_destroy(parser->decoders);
    }

    mk_list_del(&parser->_head);
    flb_free(parser);
}

void flb_parser_exit(struct flb_config *config)
{
    struct mk_list *tmp;
    struct mk_list *head;
    struct flb_parser *parser;

    /* release 'parsers' */
    mk_list_foreach_safe(head, tmp, &config->parsers) {
        parser = mk_list_entry(head, struct flb_parser, _head);
        flb_parser_destroy(parser);
    }

    /* release 'multiline parsers' */
    flb_ml_exit(config);
}

static int proc_types_str(const char *types_str, struct flb_parser_types **types)
{
    int i = 0;
    int types_num = 0;
    char *type_str = NULL;
    size_t len;
    struct mk_list *split;
    struct mk_list *head;
    struct flb_split_entry *sentry;

    split = flb_utils_split(types_str, ' ', 256);
    types_num = mk_list_size(split);
    *types = flb_malloc(sizeof(struct flb_parser_types) * types_num);

    for(i=0; i<types_num; i++){
        (*types)[i].key = NULL;
        (*types)[i].type = FLB_PARSER_TYPE_STRING;
    }
    i = 0;
    mk_list_foreach(head ,split) {
        sentry = mk_list_entry(head, struct flb_split_entry ,_head);
        type_str = strchr(sentry->value ,':');

        if (type_str == NULL) {
            i++;
            continue;
        }
        len = type_str - sentry->value;
        (*types)[i].key = flb_strndup(sentry->value, len);
        (*types)[i].key_len = len;

        type_str++;
        if (!strcasecmp(type_str, "integer")) {
            (*types)[i].type = FLB_PARSER_TYPE_INT;
        }
        else if(!strcasecmp(type_str, "bool")) {
            (*types)[i].type = FLB_PARSER_TYPE_BOOL;
        }
        else if(!strcasecmp(type_str, "float")){
            (*types)[i].type = FLB_PARSER_TYPE_FLOAT;
        }
        else if(!strcasecmp(type_str, "hex")){
            (*types)[i].type = FLB_PARSER_TYPE_HEX;
        }
        else {
            (*types)[i].type = FLB_PARSER_TYPE_STRING;
        }
        i++;
    }
    flb_utils_split_free(split);

    return i;
}

static flb_sds_t get_parser_key(struct flb_config *config,
                                struct flb_cf *cf, struct flb_cf_section *s,
                                char *key)

{
    char *tmp;
    flb_sds_t val;

    tmp = flb_cf_section_property_get(cf, s, key);
    if (!tmp) {
        return NULL;
    }

    val = flb_env_var_translate(config->env, tmp);
    if (!val) {
        return NULL;
    }

    if (flb_sds_len(val) == 0) {
        flb_sds_destroy(val);
        return NULL;
    }

    return val;
}

/* Config file: read 'parser' definitions */
static int parser_conf_file(const char *cfg, struct flb_cf *cf,
                            struct flb_config *config)
{
    flb_sds_t name;
    flb_sds_t format;
    flb_sds_t regex;
    flb_sds_t time_fmt;
    flb_sds_t time_key;
    flb_sds_t time_offset;
    flb_sds_t types_str;
    flb_sds_t tmp_str;
    int skip_empty;
    int time_keep;
    int time_strict;
    int types_len;
    struct mk_list *head;
    struct mk_list *decoders = NULL;
    struct flb_cf_section *s;
    struct flb_parser_types *types = NULL;

    /* Read all 'parser' sections */
    mk_list_foreach(head, &cf->parsers) {
        name = NULL;
        format = NULL;
        regex = NULL;
        time_fmt = NULL;
        time_key = NULL;
        time_offset = NULL;
        types_str = NULL;
        tmp_str = NULL;

        /* retrieve the section context */
        s = mk_list_entry(head, struct flb_cf_section, _head_section);

        /* name */
        name = get_parser_key(config, cf, s, "name");
        if (!name) {
            flb_error("[parser] no parser 'name' found in file '%s'", cfg);
            goto fconf_error;
        }

        /* format */
        format = get_parser_key(config, cf, s, "format");
        if (!format) {
            flb_error("[parser] no parser 'format' found for '%s' in file '%s'",
                      name, cfg);
            goto fconf_error;
        }

        /* regex (if 'format' == 'regex') */
        regex = get_parser_key(config, cf, s, "regex");
        if (!regex && strcmp(format, "regex") == 0) {
            flb_error("[parser] no parser 'regex' found for '%s' in file '%s",
                      name, cfg);
            goto fconf_error;
        }
        
        /* skip_empty_values */
        skip_empty = FLB_TRUE;
        tmp_str = get_parser_key(config, cf, s, "skip_empty_values");
        if (tmp_str) {
            skip_empty = flb_utils_bool(tmp_str);
            flb_sds_destroy(tmp_str);
        }

        /* time_format */
        time_fmt = get_parser_key(config, cf, s, "time_format");

        /* time_key */
        time_key = get_parser_key(config, cf, s, "time_key");

        /* time_keep */
        time_keep = FLB_FALSE;
        tmp_str = get_parser_key(config, cf, s, "time_keep");
        if (tmp_str) {
            time_keep = flb_utils_bool(tmp_str);
            flb_sds_destroy(tmp_str);
        }

        /* time_strict */
        time_strict = FLB_TRUE;
        tmp_str = get_parser_key(config, cf, s, "time_strict");
        if (tmp_str) {
            time_strict = flb_utils_bool(tmp_str);
            flb_sds_destroy(tmp_str);
        }

        /* time_offset (UTC offset) */
        time_offset = get_parser_key(config, cf, s, "time_offset");

        /* types */
        types_str = get_parser_key(config, cf, s, "types");
        if (types_str) {
            types_len = proc_types_str(types_str, &types);
        }
        else {
            types_len = 0;
        }

        /* Decoders */
        decoders = flb_parser_decoder_list_create(s);

        /* Create the parser context */
        if (!flb_parser_create(name, format, regex, skip_empty,
                               time_fmt, time_key, time_offset, time_keep, time_strict,
                               types, types_len, decoders, config)) {
            goto fconf_error;
        }

        flb_debug("[parser] new parser registered: %s", name);

        flb_sds_destroy(name);
        flb_sds_destroy(format);

        if (regex) {
            flb_sds_destroy(regex);
        }
        if (time_fmt) {
            flb_sds_destroy(time_fmt);
        }
        if (time_key) {
            flb_sds_destroy(time_key);
        }
        if (time_offset) {
            flb_sds_destroy(time_offset);
        }
        if (types_str) {
            flb_sds_destroy(types_str);
        }
        decoders = NULL;
    }

    return 0;

 fconf_error:
    flb_sds_destroy(name);
    flb_sds_destroy(format);
    if (regex) {
        flb_sds_destroy(regex);
    }
    if (time_fmt) {
        flb_sds_destroy(time_fmt);
    }
    if (time_key) {
        flb_sds_destroy(time_key);
    }
    if (time_offset) {
        flb_sds_destroy(time_offset);
    }
    if (types_str) {
        flb_sds_destroy(types_str);
    }
    if (types_len) {
        for (int i=0; i<types_len; i++){
	    if (types[i].key != NULL) {
                flb_free(types[i].key);
	    }
        }
        flb_free(types);
    }
    if (decoders) {
        flb_parser_decoder_list_destroy(decoders);
    }
    return -1;
}

static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser,
                                      struct flb_cf_section *section,
                                      struct flb_config *config)
{
    int ret;
    char *to_state = NULL;
    struct mk_list list;
    struct mk_list *head;
    struct flb_kv *entry;
    struct flb_slist_entry *from_state;
    struct flb_slist_entry *regex_pattern;
    struct flb_slist_entry *tmp;

    mk_list_foreach(head, &section->properties) {
        entry = mk_list_entry(head, struct flb_kv, _head);

        /* only process 'rule' keys */
        if (strcasecmp(entry->key, "rule") != 0) {
{'fraction_non_alphanumeric': 0.11842317407427312, 'fraction_numerical': 0.02265276508840759, 'mean_word_length': 1.905152224824356, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 225, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 14, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1628846', 'n_tokens_mistral': 13260, 'n_tokens_neox': 12498, 'n_words': 2683}