<commit_before>#include <pcre.h>
#include <pthread.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include <sys/time.h>
#include <unistd.h>

#include "config.h"

#include "log.h"
#include "options.h"
#include "search.h"

pthread_t *workers = NULL;
int workers_len = 0;

int main(int argc, char **argv) {
    set_log_level(LOG_LEVEL_WARN);

    char **paths = NULL;
    int i;
    int pcre_opts = PCRE_MULTILINE;
    int study_opts = 0;
    const char *pcre_err = NULL;
    int pcre_err_offset = 0;
    double time_diff = 0.0;

    /* What's the point of an init function if it's going to be 4 lines? */
    work_queue = NULL;
    workers_len = (int)sysconf(_SC_NPROCESSORS_ONLN);
    workers = calloc(workers_len, sizeof(pthread_t));
    memset(&stats, 0, sizeof(stats));
    root_ignores = init_ignore(NULL);
#ifdef USE_PCRE_JIT
    int has_jit = 0;
    pcre_config(PCRE_CONFIG_JIT, &has_jit);
    if (has_jit) {
        study_opts |= PCRE_STUDY_JIT_COMPILE;
    }
#endif

    gettimeofday(&(stats.time_start), NULL);
    log_debug("PCRE Version: %s", pcre_version());

    parse_options(argc, argv, &paths);

    if (opts.casing == CASE_INSENSITIVE) {
        pcre_opts = pcre_opts | PCRE_CASELESS;
    }

    if (opts.literal) {
        generate_skip_lookup(opts.query, opts.query_len, skip_lookup, opts.casing == CASE_SENSITIVE);
    }
    else {
        if (opts.word_regexp) {
            opts.query_len = opts.query_len + 5; /* "\b" + "\b" + '\0' */
            char *word_regexp_query = malloc(opts.query_len);
            char *word_sep = "\\b";
            strlcpy(word_regexp_query, word_sep, opts.query_len);
            strlcat(word_regexp_query, opts.query, opts.query_len);
            strlcat(word_regexp_query, word_sep, opts.query_len);
            free(opts.query);
            opts.query = word_regexp_query;
            log_debug("Word regexp query: %s", opts.query);
        }
        opts.re = pcre_compile(opts.query, pcre_opts, &pcre_err, &pcre_err_offset, NULL);
        if (opts.re == NULL) {
            log_err("pcre_compile failed at position %i. Error: %s", pcre_err_offset, pcre_err);
            exit(1);
        }
        opts.re_extra = pcre_study(opts.re, study_opts, &pcre_err);
        if (opts.re_extra == NULL) {
            log_debug("pcre_study returned nothing useful. Error: %s", pcre_err);
        }<commit_msg>Call log_debug after parsing options.<commit_after>#include <pcre.h>
#include <pthread.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include <sys/time.h>
#include <unistd.h>

#include "config.h"

#include "log.h"
#include "options.h"
#include "search.h"

pthread_t *workers = NULL;
int workers_len = 0;

int main(int argc, char **argv) {
    set_log_level(LOG_LEVEL_WARN);

    char **paths = NULL;
    int i;
    int pcre_opts = PCRE_MULTILINE;
    int study_opts = 0;
    const char *pcre_err = NULL;
    int pcre_err_offset = 0;
    double time_diff = 0.0;

    /* What's the point of an init function if it's going to be 4 lines? */
    work_queue = NULL;
    workers_len = (int)sysconf(_SC_NPROCESSORS_ONLN);
    workers = calloc(workers_len, sizeof(pthread_t));
    memset(&stats, 0, sizeof(stats));
    root_ignores = init_ignore(NULL);
#ifdef USE_PCRE_JIT
    int has_jit = 0;
    pcre_config(PCRE_CONFIG_JIT, &has_jit);
    if (has_jit) {
        study_opts |= PCRE_STUDY_JIT_COMPILE;
    }
#endif

    gettimeofday(&(stats.time_start), NULL);

    parse_options(argc, argv, &paths);
    log_debug("PCRE Version: %s", pcre_version());
    log_debug("Using %i workers", workers_len);

    if (opts.casing == CASE_INSENSITIVE) {
        pcre_opts = pcre_opts | PCRE_CASELESS;
    }

    if (opts.literal) {
        generate_skip_lookup(opts.query, opts.query_len, skip_lookup, opts.casing == CASE_SENSITIVE);
    }
    else {
        if (opts.word_regexp) {
            opts.query_len = opts.query_len + 5; /* "\b" + "\b" + '\0' */
            char *word_regexp_query = malloc(opts.query_len);
            char *word_sep = "\\b";
            strlcpy(word_regexp_query, word_sep, opts.query_len);
            strlcat(word_regexp_query, opts.query, opts.query_len);
            strlcat(word_regexp_query, word_sep, opts.query_len);
            free(opts.query);
            opts.query = word_regexp_query;
            log_debug("Word regexp query: %s", opts.query);
        }
        opts.re = pcre_compile(opts.query, pcre_opts, &pcre_err, &pcre_err_offset, NULL);
        if (opts.re == NULL) {
            log_err("pcre_compile failed at position %i. Error: %s", pcre_err_offset, pcre_err);
            exit(1);
        }
        opts.re_extra = pcre_study(opts.re, study_opts, &pcre_err);
        if (opts.re_extra == NULL) {
            log_debug("pcre_study returned nothing useful. Error: %s", pcre_err);
        }
{'fraction_non_alphanumeric': 0.13086830078936434, 'fraction_numerical': 0.004570004154549232, 'mean_word_length': 2.8458466453674123, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3127081', 'n_tokens_mistral': 1807, 'n_tokens_neox': 1758, 'n_words': 368}