<commit_before>#include <pcre.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include <sys/time.h>

#include "log.h"
#include "options.h"
#include "search.h"

#include "config.h"

int main(int argc, char **argv) {
    set_log_level(LOG_LEVEL_WARN);

    char **paths = NULL;
    int i;
    int pcre_opts = PCRE_MULTILINE;
    int study_opts = 0;
    const char *pcre_err = NULL;
    int pcre_err_offset = 0;
    pcre *re = NULL;
    pcre_extra *re_extra = NULL;
    double time_diff = 0.0;

    memset(&stats, 0, sizeof(stats)); /* What's the point of an init function if it's going to be this one-liner? */

    gettimeofday(&(stats.time_start), NULL);

    parse_options(argc, argv, &paths);

    log_debug("PCRE Version: %s", pcre_version());

    if (opts.casing == CASE_INSENSITIVE) {
        pcre_opts = pcre_opts | PCRE_CASELESS;
    }

    if (!is_regex(opts.query)) {
        /* No special chars. Do a literal match */
        opts.literal = 1;
    }

    if (opts.literal) {
        generate_skip_lookup(opts.query, opts.query_len, skip_lookup, opts.casing == CASE_SENSITIVE);
    }
    else {
        if (opts.word_regexp) {
            int query_len = opts.query_len + 5; /* "\b" + "\b" + \0 */
            char *word_regexp_query = malloc(query_len);
            char *word_sep = "\\b";
            strlcpy(word_regexp_query, word_sep, query_len);
            strlcat(word_regexp_query, opts.query, query_len);
            strlcat(word_regexp_query, word_sep, query_len);
            free(opts.query);
            opts.query = word_regexp_query;
            opts.query_len = query_len;
            log_debug("Word regexp query: %s", opts.query);
        }
        re = pcre_compile(opts.query, pcre_opts, &pcre_err, &pcre_err_offset, NULL);
        if (re == NULL) {
            log_err("pcre_compile failed at position %i. Error: %s", pcre_err_offset, pcre_err);
            exit(1);
        }
<commit_msg>Don't really need this variable.<commit_after>#include <pcre.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include <sys/time.h>

#include "log.h"
#include "options.h"
#include "search.h"

#include "config.h"

int main(int argc, char **argv) {
    set_log_level(LOG_LEVEL_WARN);

    char **paths = NULL;
    int i;
    int pcre_opts = PCRE_MULTILINE;
    int study_opts = 0;
    const char *pcre_err = NULL;
    int pcre_err_offset = 0;
    pcre *re = NULL;
    pcre_extra *re_extra = NULL;
    double time_diff = 0.0;

    memset(&stats, 0, sizeof(stats)); /* What's the point of an init function if it's going to be this one-liner? */

    gettimeofday(&(stats.time_start), NULL);

    parse_options(argc, argv, &paths);

    log_debug("PCRE Version: %s", pcre_version());

    if (opts.casing == CASE_INSENSITIVE) {
        pcre_opts = pcre_opts | PCRE_CASELESS;
    }

    if (!is_regex(opts.query)) {
        /* No special chars. Do a literal match */
        opts.literal = 1;
    }

    if (opts.literal) {
        generate_skip_lookup(opts.query, opts.query_len, skip_lookup, opts.casing == CASE_SENSITIVE);
    }
    else {
        if (opts.word_regexp) {
            opts.query_len = opts.query_len + 5; /* "\b" + "\b" + '\0' */
            char *word_regexp_query = malloc(opts.query_len);
            char *word_sep = "\\b";
            strlcpy(word_regexp_query, word_sep, opts.query_len);
            strlcat(word_regexp_query, opts.query, opts.query_len);
            strlcat(word_regexp_query, word_sep, opts.query_len);
            free(opts.query);
            opts.query = word_regexp_query;
            log_debug("Word regexp query: %s", opts.query);
        }
        re = pcre_compile(opts.query, pcre_opts, &pcre_err, &pcre_err_offset, NULL);
        if (re == NULL) {
            log_err("pcre_compile failed at position %i. Error: %s", pcre_err_offset, pcre_err);
            exit(1);
        }

{'fraction_non_alphanumeric': 0.13326446280991736, 'fraction_numerical': 0.0046487603305785125, 'mean_word_length': 2.553211009174312, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4683442', 'n_tokens_mistral': 1417, 'n_tokens_neox': 1369, 'n_words': 311}