<commit_before>#include <dirent.h>
#include <fnmatch.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "ignore.h"
#include "log.h"
#include "options.h"
#include "util.h"

/* TODO: build a huge-ass list of files we want to ignore by default (build cache stuff, pyc files, etc) */

const char *evil_hardcoded_ignore_files[] = {
    ".",
    "..",
    NULL
};

const char *ignore_pattern_files[] = {
    ".agignore",
    ".gitignore",
    ".hgignore",
    ".svn",
    NULL
};

/* For patterns that need fnmatch */
char **ignore_patterns = NULL;
int ignore_patterns_len = 0;

/* For patterns with no regex stuff in them. Sorted for fast matching. */
char **ignore_names = NULL;
int ignore_names_len = 0;

const int fnmatch_flags = 0 & FNM_PATHNAME;

void add_ignore_pattern(const char* pattern) {
    int i;

    if (is_fnmatch(pattern)) {
        ignore_patterns_len++;
        ignore_patterns = realloc(ignore_patterns, (ignore_patterns_len) * sizeof(char*));
        ignore_patterns[ignore_patterns_len - 1] = strdup(pattern);
        log_debug("added regex ignore pattern %s", pattern);
    }
    else {
        /* a balanced binary tree is best for performance, but I'm lazy */
        ignore_names_len++;
        ignore_names = realloc(ignore_names, ignore_names_len * sizeof(char*));
        for (i = ignore_names_len-1; i > 0; i--) {
            if (strcmp(pattern, ignore_names[i-1]) > 0) {
                break;
            }
            ignore_names[i] = ignore_names[i-1];
        }
        ignore_names[i] = strdup(pattern);
        log_debug("added literal ignore pattern %s", pattern);
    }
}

void cleanup_ignore_patterns() {
    int i;
    for (i = 0; i<ignore_patterns_len; i++) {
        free(ignore_patterns[i]);
    }
    free(ignore_patterns);

    for (i = 0; i<ignore_names_len; i++) {
        free(ignore_names[i]);
    }
    free(ignore_names);
}

void load_svn_ignore_patterns(const char *path, const int path_len) {
    FILE *fp = NULL;
    char *dir_prop_base = malloc(path_len + strlen(SVN_DIR_PROP_BASE) + 1);
    strlcpy(dir_prop_base, path, path_len + 1);
    strlcat(dir_prop_base, SVN_DIR_PROP_BASE, path_len + strlen(SVN_DIR_PROP_BASE) + 1);

    fp = fopen(dir_prop_base, "r");
    if (fp == NULL) {
        log_debug("Skipping svn ignore file %s", dir_prop_base);
        return;
    }

    char *entry = NULL;
    size_t entry_len = 0;
    char *key = malloc(32);
    size_t key_len = 0;
    size_t bytes_read = 0;
    char *entry_line = NULL;
    size_t line_len;

    while (fscanf(fp, "K %zu\n", &key_len) == 1) {
        key = realloc(key, (key_len + 1) * sizeof(char));
        bytes_read = fread(key, 1, key_len, fp);
        key[key_len] = '\0';
        fscanf(fp, "\nV %zu\n", &entry_len); /* TODO: make sure fscanf worked */

        if (strncmp(SVN_PROP_IGNORE, key, bytes_read) != 0) {
            log_err("key is %s, not %s. skipping %u bytes", key, SVN_PROP_IGNORE, entry_len);
            /* Not the key we care about. fseek and repeat */
            fseek(fp, entry_len + 1, SEEK_CUR); /* +1 to account for newline. yes I know this is hacky */
            continue;
        }
        /* Aww yeah. Time to ignore stuff */
        entry = malloc(entry_len + 1);
        bytes_read = fread(entry, 1, entry_len, fp);
        entry[bytes_read] = '\0';
        log_err("entry: %s", entry);
        break;
    }
    char *patterns = entry;
    while (*patterns != '\0' && patterns < (entry + bytes_read)) {
        for (line_len = 0; line_len < strlen(patterns); line_len++) {
            if (patterns[line_len] == '\n') {
                break;
            }
        }
        if (line_len > 0) {
            entry_line = malloc((size_t)line_len + 1);
            strlcpy(entry_line, patterns, line_len + 1);
            add_ignore_pattern(entry_line);
            free(entry_line);
            entry_line = NULL;
        }
        patterns += line_len + 1;
    }
    free(entry);
    free(key);
    fclose(fp);
}

/* For loading git/svn/hg ignore patterns */
void load_ignore_patterns(const char *ignore_filename) {
    FILE *fp = NULL;
    fp = fopen(ignore_filename, "r");
    if (fp == NULL) {
        log_debug("Skipping ignore file %s", ignore_filename);
        return;
    }

    char *line = NULL;
    ssize_t line_len = 0;
    size_t line_cap = 0;

    while ((line_len = getline(&line, &line_cap, fp)) > 0) {
        if (line_len == 0 || line[0] == '\n') {
            continue;
        }
        if (line[line_len-1] == '\n') {
            line[line_len-1] = '\0'; /* kill the \n */
        }
        add_ignore_pattern(line);
    }

    free(line);<commit_msg>Fix more logging. Don't try to free or dereference stuff that's null.<commit_after>#include <dirent.h>
#include <fnmatch.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "ignore.h"
#include "log.h"
#include "options.h"
#include "util.h"

/* TODO: build a huge-ass list of files we want to ignore by default (build cache stuff, pyc files, etc) */

const char *evil_hardcoded_ignore_files[] = {
    ".",
    "..",
    NULL
};

const char *ignore_pattern_files[] = {
    ".agignore",
    ".gitignore",
    ".hgignore",
    ".svn",
    NULL
};

/* For patterns that need fnmatch */
char **ignore_patterns = NULL;
int ignore_patterns_len = 0;

/* For patterns with no regex stuff in them. Sorted for fast matching. */
char **ignore_names = NULL;
int ignore_names_len = 0;

const int fnmatch_flags = 0 & FNM_PATHNAME;

void add_ignore_pattern(const char* pattern) {
    int i;

    if (is_fnmatch(pattern)) {
        ignore_patterns_len++;
        ignore_patterns = realloc(ignore_patterns, (ignore_patterns_len) * sizeof(char*));
        ignore_patterns[ignore_patterns_len - 1] = strdup(pattern);
        log_debug("added regex ignore pattern %s", pattern);
    }
    else {
        /* a balanced binary tree is best for performance, but I'm lazy */
        ignore_names_len++;
        ignore_names = realloc(ignore_names, ignore_names_len * sizeof(char*));
        for (i = ignore_names_len-1; i > 0; i--) {
            if (strcmp(pattern, ignore_names[i-1]) > 0) {
                break;
            }
            ignore_names[i] = ignore_names[i-1];
        }
        ignore_names[i] = strdup(pattern);
        log_debug("added literal ignore pattern %s", pattern);
    }
}

void cleanup_ignore_patterns() {
    int i;
    for (i = 0; i<ignore_patterns_len; i++) {
        free(ignore_patterns[i]);
    }
    free(ignore_patterns);

    for (i = 0; i<ignore_names_len; i++) {
        free(ignore_names[i]);
    }
    free(ignore_names);
}

void load_svn_ignore_patterns(const char *path, const int path_len) {
    FILE *fp = NULL;
    char *dir_prop_base = malloc(path_len + strlen(SVN_DIR_PROP_BASE) + 1);
    strlcpy(dir_prop_base, path, path_len + 1);
    strlcat(dir_prop_base, SVN_DIR_PROP_BASE, path_len + strlen(SVN_DIR_PROP_BASE) + 1);

    fp = fopen(dir_prop_base, "r");
    if (fp == NULL) {
        log_debug("Skipping svn ignore file %s", dir_prop_base);
        return;
    }

    char *entry = NULL;
    size_t entry_len = 0;
    char *key = malloc(32);
    size_t key_len = 0;
    size_t bytes_read = 0;
    char *entry_line = NULL;
    size_t line_len;

    while (fscanf(fp, "K %zu\n", &key_len) == 1) {
        key = realloc(key, (key_len + 1) * sizeof(char));
        bytes_read = fread(key, 1, key_len, fp);
        key[key_len] = '\0';
        fscanf(fp, "\nV %zu\n", &entry_len); /* TODO: make sure fscanf worked */

        if (strncmp(SVN_PROP_IGNORE, key, bytes_read) != 0) {
            log_debug("key is %s, not %s. skipping %u bytes", key, SVN_PROP_IGNORE, entry_len);
            /* Not the key we care about. fseek and repeat */
            fseek(fp, entry_len + 1, SEEK_CUR); /* +1 to account for newline. yes I know this is hacky */
            continue;
        }
        /* Aww yeah. Time to ignore stuff */
        entry = malloc(entry_len + 1);
        bytes_read = fread(entry, 1, entry_len, fp);
        entry[bytes_read] = '\0';
        log_debug("entry: %s", entry);
        break;
    }
    if (entry == NULL) {
        goto cleanup;
    }
    char *patterns = entry;
    while (*patterns != '\0' && patterns < (entry + bytes_read)) {
        for (line_len = 0; line_len < strlen(patterns); line_len++) {
            if (patterns[line_len] == '\n') {
                break;
            }
        }
        if (line_len > 0) {
            entry_line = malloc((size_t)line_len + 1);
            strlcpy(entry_line, patterns, line_len + 1);
            add_ignore_pattern(entry_line);
            free(entry_line);
            entry_line = NULL;
        }
        patterns += line_len + 1;
    }
    free(entry);
    cleanup:
    free(key);
    fclose(fp);
}

/* For loading git/svn/hg ignore patterns */
void load_ignore_patterns(const char *ignore_filename) {
    FILE *fp = NULL;
    fp = fopen(ignore_filename, "r");
    if (fp == NULL) {
        log_debug("Skipping ignore file %s", ignore_filename);
        return;
    }

    char *line = NULL;
    ssize_t line_len = 0;
    size_t line_cap = 0;

    while ((line_len = getline(&line, &line_cap, fp)) > 0) {
        if (line_len == 0 || line[0] == '\n') {
            continue;
        }
        if (line[line_len-1] == '\n') {
            line[line_len-1] = '\0'; /* kill the \n */
        }
        add_ignore_pattern(line);
    }

    free(line);
{'fraction_non_alphanumeric': 0.13796768707482993, 'fraction_numerical': 0.009141156462585034, 'mean_word_length': 2.3808839381961913, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4280843', 'n_tokens_mistral': 3355, 'n_tokens_neox': 3246, 'n_words': 816}