<commit_before>/*
 * Copyright (c) 2014,2015 KLab Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <getopt.h>
#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <regex.h>
#include <sys/stat.h>
#include <sys/queue.h>
#include <ev.h>
#include <pcre.h>
#include "common.h"
#include "rlogd.h"

#define DEFAULT_CONFIG_FILE (SYSCONFDIR "/rlogd.conf")
#define DEFAULT_PID_FILE (LOCALSTATEDIR "/run/rlogd.pid")

int __dryrun;

typedef struct {
    char *config;
    int debug;
    int dryrun;
    int foreground;
    char *pid;
    char *_stdout;
    char *_stderr;
} option_t;

struct signal_def {
    int signum;
    struct ev_signal w;
};

static struct signal_def signals[] = {
    {.signum = SIGINT },
    {.signum = SIGTERM},
    {.signum = 0}
};

struct source {
    struct module ctx;
    char *label;
    char *prefix;
    char *suffix;
    pthread_t thread;
    TAILQ_ENTRY(source) lp;
};

struct match {
    struct module ctx;
    pcre *reg;
    pthread_t thread;
    pthread_mutex_t mutex;
    TAILQ_ENTRY(match) lp;
};

struct label {
    pcre *reg;
    TAILQ_HEAD(/**/, match) matches;
    TAILQ_ENTRY(label) lp;
};

#define MODULE_TYPE_INPUT  (1)
#define MODULE_TYPE_OUTPUT (2)

#define MODULE_DEFINE(XX)                                \
    XX(MODULE_TYPE_INPUT,  "forward", in_forward_setup)  \
    XX(MODULE_TYPE_OUTPUT, "forward", out_forward_setup) \
    XX(MODULE_TYPE_OUTPUT, "file",    out_file_setup)    \

#define MODULE_DECLARE(type, name, func) extern int func (struct module *module, const struct dir *dir);
MODULE_DEFINE(MODULE_DECLARE)
#undef  MODULE_DECLARE 

struct module_def {
    int type;
    char *name;
    int (*setup)(struct module *, const struct dir *);
};

static struct module_def modules[] = {
#define MODULE_DECLARE(type, name, func) {type, name, func},
    MODULE_DEFINE(MODULE_DECLARE)
#undef  MODULE_DECLARE
    {0, NULL, NULL}
};

#undef MODULE_DEFINE

TAILQ_HEAD(/**/, source) sources = TAILQ_HEAD_INITIALIZER(sources);
TAILQ_HEAD(/**/, match)  matches = TAILQ_HEAD_INITIALIZER(matches);
TAILQ_HEAD(/**/, label)  labels  = TAILQ_HEAD_INITIALIZER(labels);

void
push_entries (struct module *module, const char *tag, size_t tag_len, struct entry *entries, size_t len) {
    struct source *source;
    char buf[1024];
    size_t n = 0;
    struct label *label;
    struct match *match;

    source = container_of(module, struct source, ctx);
    if (!source) {
        return;
    }
    if (source->prefix) {
        strcpy(buf, source->prefix);
        n += strlen(source->prefix);
        buf[n++] = '.';
    }
    strncpy(buf + n, tag, tag_len);
    n += tag_len;
    if (source->suffix) {
        buf[n++] = '.';
        strcpy(buf + n, source->suffix);
    } else {
        buf[n] = '\0';
    }
    if (source->label) {
        TAILQ_FOREACH(label, &labels, lp) {
            if (pcre_exec(label->reg, NULL, source->label, strlen(source->label), 0, 0, 0, 0) >= 0) {
                TAILQ_FOREACH(match, &label->matches, lp) {
                    if (pcre_exec(match->reg, NULL, buf, strlen(buf), 0, 0, 0, 0) >= 0) {
                        pthread_mutex_lock(&match->mutex);
                        match->ctx.emit(match->ctx.arg, buf, n, entries, len);
                        pthread_mutex_unlock(&match->mutex);
                        return;
                    }
                }
                warning_print("'%.*s' is not match", (int)tag_len, tag);
                return;
            }
        }
        warning_print("label '%s' is not found", source->label);
        return;
    }
    TAILQ_FOREACH(match, &matches, lp) {
        if (pcre_exec(match->reg, NULL, buf, strlen(buf), 0, 0, 0, 0) >= 0) {
            pthread_mutex_lock(&match->mutex);
            match->ctx.emit(match->ctx.arg, buf, n, entries, len);
            pthread_mutex_unlock(&match->mutex);
            return;
        }
    }
    warning_print("'%.*s' is not match", (int)tag_len, tag);
}

void
cancel_modules (void) {
    struct source *source;
    struct match *match;
    struct label *label;

    while ((source = TAILQ_FIRST(&sources)) != NULL) {
        source->ctx.cancel(source->ctx.arg);
        pthread_join(source->thread, NULL);
        TAILQ_REMOVE(&sources, source, lp);
        free(source);
    }
    while ((match = TAILQ_FIRST(&matches)) != NULL) {
        match->ctx.cancel(match->ctx.arg);
        pthread_join(match->thread, NULL);
        TAILQ_REMOVE(&matches, match, lp);
        pcre_free(match->reg);
        free(match);
    }
    while ((label = TAILQ_FIRST(&labels)) != NULL) {
        while ((match = TAILQ_FIRST(&label->matches)) != NULL) {
            match->ctx.cancel(match->ctx.arg);
            pthread_join(match->thread, NULL);
            TAILQ_REMOVE(&matches, match, lp);
            pcre_free(match->reg);
            free(match);
        }
        TAILQ_REMOVE(&labels, label, lp);
        pcre_free(label->reg);
        free(label);
    }
}

void
run_modules (void) {
    struct label *label;
    struct match *match;
    struct source *source;

    TAILQ_FOREACH(label, &labels, lp) {
        TAILQ_FOREACH(match, &label->matches, lp) {
            pthread_create(&match->thread, NULL, match->ctx.run, match->ctx.arg);
        }
    }
    TAILQ_FOREACH(match, &matches, lp) {
        pthread_create(&match->thread, NULL, match->ctx.run, match->ctx.arg);
    }
    TAILQ_FOREACH(source, &sources, lp) {
        pthread_create(&source->thread, NULL, source->ctx.run, source->ctx.arg);
    }
}<commit_msg>Fixed long time required to shutdown<commit_after>/*
 * Copyright (c) 2014,2015 KLab Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <getopt.h>
#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <regex.h>
#include <sys/stat.h>
#include <sys/queue.h>
#include <ev.h>
#include <pcre.h>
#include "common.h"
#include "rlogd.h"

#define DEFAULT_CONFIG_FILE (SYSCONFDIR "/rlogd.conf")
#define DEFAULT_PID_FILE (LOCALSTATEDIR "/run/rlogd.pid")

int __dryrun;

typedef struct {
    char *config;
    int debug;
    int dryrun;
    int foreground;
    char *pid;
    char *_stdout;
    char *_stderr;
} option_t;

struct signal_def {
    int signum;
    struct ev_signal w;
};

static struct signal_def signals[] = {
    {.signum = SIGINT },
    {.signum = SIGTERM},
    {.signum = 0}
};

struct source {
    struct module ctx;
    char *label;
    char *prefix;
    char *suffix;
    pthread_t thread;
    TAILQ_ENTRY(source) lp;
};

struct match {
    struct module ctx;
    pcre *reg;
    pthread_t thread;
    pthread_mutex_t mutex;
    TAILQ_ENTRY(match) lp;
};

struct label {
    pcre *reg;
    TAILQ_HEAD(/**/, match) matches;
    TAILQ_ENTRY(label) lp;
};

#define MODULE_TYPE_INPUT  (1)
#define MODULE_TYPE_OUTPUT (2)

#define MODULE_DEFINE(XX)                                \
    XX(MODULE_TYPE_INPUT,  "forward", in_forward_setup)  \
    XX(MODULE_TYPE_OUTPUT, "forward", out_forward_setup) \
    XX(MODULE_TYPE_OUTPUT, "file",    out_file_setup)    \

#define MODULE_DECLARE(type, name, func) extern int func (struct module *module, const struct dir *dir);
MODULE_DEFINE(MODULE_DECLARE)
#undef  MODULE_DECLARE 

struct module_def {
    int type;
    char *name;
    int (*setup)(struct module *, const struct dir *);
};

static struct module_def modules[] = {
#define MODULE_DECLARE(type, name, func) {type, name, func},
    MODULE_DEFINE(MODULE_DECLARE)
#undef  MODULE_DECLARE
    {0, NULL, NULL}
};

#undef MODULE_DEFINE

TAILQ_HEAD(/**/, source) sources = TAILQ_HEAD_INITIALIZER(sources);
TAILQ_HEAD(/**/, match)  matches = TAILQ_HEAD_INITIALIZER(matches);
TAILQ_HEAD(/**/, label)  labels  = TAILQ_HEAD_INITIALIZER(labels);

void
push_entries (struct module *module, const char *tag, size_t tag_len, struct entry *entries, size_t len) {
    struct source *source;
    char buf[1024];
    size_t n = 0;
    struct label *label;
    struct match *match;

    source = container_of(module, struct source, ctx);
    if (!source) {
        return;
    }
    if (source->prefix) {
        strcpy(buf, source->prefix);
        n += strlen(source->prefix);
        buf[n++] = '.';
    }
    strncpy(buf + n, tag, tag_len);
    n += tag_len;
    if (source->suffix) {
        buf[n++] = '.';
        strcpy(buf + n, source->suffix);
    } else {
        buf[n] = '\0';
    }
    if (source->label) {
        TAILQ_FOREACH(label, &labels, lp) {
            if (pcre_exec(label->reg, NULL, source->label, strlen(source->label), 0, 0, 0, 0) >= 0) {
                TAILQ_FOREACH(match, &label->matches, lp) {
                    if (pcre_exec(match->reg, NULL, buf, strlen(buf), 0, 0, 0, 0) >= 0) {
                        pthread_mutex_lock(&match->mutex);
                        match->ctx.emit(match->ctx.arg, buf, n, entries, len);
                        pthread_mutex_unlock(&match->mutex);
                        return;
                    }
                }
                warning_print("'%.*s' is not match", (int)tag_len, tag);
                return;
            }
        }
        warning_print("label '%s' is not found", source->label);
        return;
    }
    TAILQ_FOREACH(match, &matches, lp) {
        if (pcre_exec(match->reg, NULL, buf, strlen(buf), 0, 0, 0, 0) >= 0) {
            pthread_mutex_lock(&match->mutex);
            match->ctx.emit(match->ctx.arg, buf, n, entries, len);
            pthread_mutex_unlock(&match->mutex);
            return;
        }
    }
    warning_print("'%.*s' is not match", (int)tag_len, tag);
}

void
cancel_modules (void) {
    struct source *source;
    struct match *match;
    struct label *label;

    TAILQ_FOREACH(source, &sources, lp) {
        source->ctx.cancel(source->ctx.arg);
    }
    while ((source = TAILQ_FIRST(&sources)) != NULL) {
        pthread_join(source->thread, NULL);
        TAILQ_REMOVE(&sources, source, lp);
        free(source);
    }
    TAILQ_FOREACH(match, &matches, lp) {
        match->ctx.cancel(match->ctx.arg);
    }
    while ((match = TAILQ_FIRST(&matches)) != NULL) {
        pthread_join(match->thread, NULL);
        TAILQ_REMOVE(&matches, match, lp);
        pcre_free(match->reg);
        free(match);
    }
    while ((label = TAILQ_FIRST(&labels)) != NULL) {
        TAILQ_FOREACH(match, &label->matches, lp) {
            match->ctx.cancel(match->ctx.arg);
        }
        while ((match = TAILQ_FIRST(&label->matches)) != NULL) {
            pthread_join(match->thread, NULL);
            TAILQ_REMOVE(&matches, match, lp);
            pcre_free(match->reg);
            free(match);
        }
        TAILQ_REMOVE(&labels, label, lp);
        pcre_free(label->reg);
        free(label);
    }
}

void
run_modules (void) {
    struct label *label;
    struct match *match;
    struct source *source;

    TAILQ_FOREACH(label, &labels, lp) {
        TAILQ_FOREACH(match, &label->matches, lp) {
            pthread_create(&match->thread, NULL, match->ctx.run, match->ctx.arg);
        }
    }
    TAILQ_FOREACH(match, &matches, lp) {
        pthread_create(&match->thread, NULL, match->ctx.run, match->ctx.arg);
    }
    TAILQ_FOREACH(source, &sources, lp) {
        pthread_create(&source->thread, NULL, source->ctx.run, source->ctx.arg);
    }
}
{'fraction_non_alphanumeric': 0.13068181818181818, 'fraction_numerical': 0.00487012987012987, 'mean_word_length': 2.7397902869757176, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 126, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6556852', 'n_tokens_mistral': 4539, 'n_tokens_neox': 4376, 'n_words': 1196}