<commit_before>/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/*  Fluent Bit
 *  ==========
 *  Copyright (C) 2015-2022 The Fluent Bit Authors
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#include <fluent-bit/flb_output_plugin.h>
#include <fluent-bit/flb_log.h>
#include <fluent-bit/flb_kv.h>
#include <fluent-bit/flb_upstream.h>
#include <fluent-bit/flb_utils.h>
#include <fluent-bit/flb_digest.h>
#include <fluent-bit/flb_pack.h>
#include <fluent-bit/flb_version.h>
#include <fluent-bit/flb_metrics.h>
#include <fluent-bit/flb_fstore.h>

#include "calyptia.h"

#include <cmetrics/cmetrics.h>
#include <cmetrics/cmt_encode_influx.h>

flb_sds_t custom_calyptia_pipeline_config_get(struct flb_config *ctx);

static int get_io_flags(struct flb_output_instance *ins)
{
    int flags = 0;

    if (ins->use_tls) {
        flags = FLB_IO_TLS;
    }
    else {
        flags = FLB_IO_TCP;
    }

    return flags;
}

static int config_add_labels(struct flb_output_instance *ins,
                             struct flb_calyptia *ctx)
{
    struct mk_list *head;
    struct flb_config_map_val *mv;
    struct flb_slist_entry *k = NULL;
    struct flb_slist_entry *v = NULL;
    struct flb_kv *kv;

    if (!ctx->add_labels || mk_list_size(ctx->add_labels) == 0) {
        return 0;
    }

    /* iterate all 'add_label' definitions */
    flb_config_map_foreach(head, mv, ctx->add_labels) {
        if (mk_list_size(mv->val.list) != 2) {
            flb_plg_error(ins, "'add_label' expects a key and a value, "
                          "e.g: 'add_label version 1.8.x'");
            return -1;
        }

        k = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head);
        v = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head);

        kv = flb_kv_item_create(&ctx->kv_labels, k->str, v->str);
        if (!kv) {
            flb_plg_error(ins, "could not append label %s=%s\n", k->str, v->str);
            return -1;
        }
    }

    return 0;
}

static void append_labels(struct flb_calyptia *ctx, struct cmt *cmt)
{
    struct flb_kv *kv;
    struct mk_list *head;

    mk_list_foreach(head, &ctx->kv_labels) {
        kv = mk_list_entry(head, struct flb_kv, _head);
        cmt_label_add(cmt, kv->key, kv->val);
    }
}

static flb_sds_t sha256_to_hex(unsigned char *sha256)
{
    int i;
    flb_sds_t hex;
    flb_sds_t tmp;

    hex = flb_sds_create_size(64);
    if (!hex) {
        return NULL;
    }

    for (i = 0; i < 32; i++) {
        tmp = flb_sds_printf(&hex, "%02x", sha256[i]);
        if (!tmp) {
            flb_sds_destroy(hex);
            return NULL;
        }
        hex = tmp;
    }

    flb_sds_len_set(hex, 64);
    return hex;
}

static int get_machine_id(struct flb_calyptia *ctx, char **out_buf, size_t *out_size)
{
    int ret;
    char *buf;
    flb_sds_t s_buf;
    size_t s;
    unsigned char sha256_buf[64] = {0};

    /* retrieve raw machine id */
    ret = flb_utils_get_machine_id(&buf, &s);
    if (ret == -1) {
        flb_plg_error(ctx->ins, "could not obtain machine id");
        return -1;
    }

    ret = flb_digest_simple(FLB_DIGEST_SHA256,
                            (unsigned char *) buf,
                            s,
                            sha256_buf,
                            sizeof(sha256_buf));

    flb_free(buf);

    if (ret != FLB_CRYPTO_SUCCESS) {
        return -1;
    }

    /* convert to hex */
    s_buf = sha256_to_hex(sha256_buf);
    if (!s_buf) {
        return -1;
    }

    *out_buf = s_buf;<commit_msg>out_calyptia: renamed flb_digest to flb_hash<commit_after>/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/*  Fluent Bit
 *  ==========
 *  Copyright (C) 2015-2022 The Fluent Bit Authors
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#include <fluent-bit/flb_output_plugin.h>
#include <fluent-bit/flb_log.h>
#include <fluent-bit/flb_kv.h>
#include <fluent-bit/flb_upstream.h>
#include <fluent-bit/flb_utils.h>
#include <fluent-bit/flb_hash.h>
#include <fluent-bit/flb_pack.h>
#include <fluent-bit/flb_version.h>
#include <fluent-bit/flb_metrics.h>
#include <fluent-bit/flb_fstore.h>

#include "calyptia.h"

#include <cmetrics/cmetrics.h>
#include <cmetrics/cmt_encode_influx.h>

flb_sds_t custom_calyptia_pipeline_config_get(struct flb_config *ctx);

static int get_io_flags(struct flb_output_instance *ins)
{
    int flags = 0;

    if (ins->use_tls) {
        flags = FLB_IO_TLS;
    }
    else {
        flags = FLB_IO_TCP;
    }

    return flags;
}

static int config_add_labels(struct flb_output_instance *ins,
                             struct flb_calyptia *ctx)
{
    struct mk_list *head;
    struct flb_config_map_val *mv;
    struct flb_slist_entry *k = NULL;
    struct flb_slist_entry *v = NULL;
    struct flb_kv *kv;

    if (!ctx->add_labels || mk_list_size(ctx->add_labels) == 0) {
        return 0;
    }

    /* iterate all 'add_label' definitions */
    flb_config_map_foreach(head, mv, ctx->add_labels) {
        if (mk_list_size(mv->val.list) != 2) {
            flb_plg_error(ins, "'add_label' expects a key and a value, "
                          "e.g: 'add_label version 1.8.x'");
            return -1;
        }

        k = mk_list_entry_first(mv->val.list, struct flb_slist_entry, _head);
        v = mk_list_entry_last(mv->val.list, struct flb_slist_entry, _head);

        kv = flb_kv_item_create(&ctx->kv_labels, k->str, v->str);
        if (!kv) {
            flb_plg_error(ins, "could not append label %s=%s\n", k->str, v->str);
            return -1;
        }
    }

    return 0;
}

static void append_labels(struct flb_calyptia *ctx, struct cmt *cmt)
{
    struct flb_kv *kv;
    struct mk_list *head;

    mk_list_foreach(head, &ctx->kv_labels) {
        kv = mk_list_entry(head, struct flb_kv, _head);
        cmt_label_add(cmt, kv->key, kv->val);
    }
}

static flb_sds_t sha256_to_hex(unsigned char *sha256)
{
    int i;
    flb_sds_t hex;
    flb_sds_t tmp;

    hex = flb_sds_create_size(64);
    if (!hex) {
        return NULL;
    }

    for (i = 0; i < 32; i++) {
        tmp = flb_sds_printf(&hex, "%02x", sha256[i]);
        if (!tmp) {
            flb_sds_destroy(hex);
            return NULL;
        }
        hex = tmp;
    }

    flb_sds_len_set(hex, 64);
    return hex;
}

static int get_machine_id(struct flb_calyptia *ctx, char **out_buf, size_t *out_size)
{
    int ret;
    char *buf;
    flb_sds_t s_buf;
    size_t s;
    unsigned char sha256_buf[64] = {0};

    /* retrieve raw machine id */
    ret = flb_utils_get_machine_id(&buf, &s);
    if (ret == -1) {
        flb_plg_error(ctx->ins, "could not obtain machine id");
        return -1;
    }

    ret = flb_hash_simple(FLB_HASH_SHA256,
                          (unsigned char *) buf,
                          s,
                          sha256_buf,
                          sizeof(sha256_buf));

    flb_free(buf);

    if (ret != FLB_CRYPTO_SUCCESS) {
        return -1;
    }

    /* convert to hex */
    s_buf = sha256_to_hex(sha256_buf);
    if (!s_buf) {
        return -1;
    }

    *out_buf = s_buf;
{'fraction_non_alphanumeric': 0.120577794099645, 'fraction_numerical': 0.016158648549394052, 'mean_word_length': 2.422706325932132, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4017140', 'n_tokens_mistral': 3123, 'n_tokens_neox': 3007, 'n_words': 707}