<commit_before>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <unistd.h>
#include <fcntl.h>

#include <discodb.h>

static void print_cursor(struct ddb *db, struct ddb_cursor *cur)
{
        if (!cur){
                const char *err;
                ddb_error(db, &err);
                fprintf(stderr, "Query failed: %s\n", err);
                exit(1);
        }

        if (ddb_empty_cursor(cur)){
                fprintf(stderr, "Not found\n");
                exit(1);
        }
        int i = 0;
        const struct ddb_entry *e;
        while ((e = ddb_next(cur))){
                printf("%.*s\n", e->length, e->data);
                ++i;
        }
        ddb_free_cursor(cur);
}

static struct ddb_query_clause *parse_cnf(char **tokens, int num, int *num_clauses)
{
        int k, i, j, t;
        *num_clauses = 1;
        for (i = 0; i < num; i++)
                if (!strcmp(tokens[i], "&"))
                        ++*num_clauses;

        struct ddb_query_term *terms = calloc(num - (*num_clauses - 1),
                sizeof(struct ddb_query_term));
        struct ddb_query_clause *clauses = calloc(*num_clauses,
                sizeof(struct ddb_query_clause));

        clauses[0].terms = terms;
        
        for (t = 0, k = 0, j = 0, i = 0; i < num; i++){
                if (!strcmp(tokens[i], "&")){
                        clauses[j++].num_terms = i - k;
                        clauses[j].terms = &terms[t];
                        k = t;
                        continue;
                }
                if (tokens[i][0] == '-'){
                        terms[t].not = 1;
                        terms[t].key.data = &tokens[i][1];
                        terms[t].key.length = strlen(&tokens[i][1]);
                }else{
                        terms[t].key.data = tokens[i];
                        terms[t].key.length = strlen(tokens[i]);
                }
                ++t;
        }
        clauses[j].num_terms = t - k;
#ifdef DEBUG
        for (i = 0; i < *num_clauses; i++){
                printf("dbg Clause:\n");
                for (j = 0; j < clauses[i].num_terms; j++){
                        if (clauses[i].terms[j].not)
                                printf("dbg NOT ");
                        printf("dbg %.*s\n", clauses[i].terms[j].key.length,
                                clauses[i].terms[j].key.data);
                }
                printf("dbg ---\n");
        }
#endif
        return clauses;
}

static struct ddb *open_discodb(const char *file)
{
        struct stat nfo;
        int fd;
        
        if ((fd = open(file, O_RDONLY)) == -1){
                fprintf(stderr, "Couldn't open discodb %s\n", file);
                exit(1);
        }
        if (fstat(fd, &nfo)){
                fprintf(stderr, "Couldn't get the file size for %s\n", file);
                exit(1);
        }
        
        const char *p = mmap(0, nfo.st_size, PROT_READ, MAP_SHARED, fd, 0);
        if (p == MAP_FAILED){
                fprintf(stderr, "Loading %s failed\n", file);
                exit(1);
        }
        struct ddb *db;
        if (!(db = ddb_new())){
                fprintf(stderr, "Couldn't initialize DiscoDB: out of memory\n");
                exit(1);
        }
        if (ddb_loads(db, p, nfo.st_size)){
                const char *err;
                ddb_error(db, &err);
                fprintf(stderr, "Invalid discodb in %s: %s\n", file, err);
                exit(1);
        }
        return db;
}

static void usage()
{
        fprintf(stderr, "Usage:\n");
        fprintf(stderr, "query_discodb [discodb] [-keys|-values|-item|-cnf] [query]\n");
        fprintf(stderr, "cnf format example: a b & -c d & e\n");
        exit(1);
}

int main(int argc, char **argv)
{
        if (argc < 3)
                usage();<commit_msg>stylistic change in error message<commit_after>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <unistd.h>
#include <fcntl.h>

#include <discodb.h>

static void print_cursor(struct ddb *db, struct ddb_cursor *cur)
{
        if (!cur){
                const char *err;
                ddb_error(db, &err);
                fprintf(stderr, "Query failed: %s\n", err);
                exit(1);
        }

        if (ddb_empty_cursor(cur)){
                fprintf(stderr, "Not found\n");
                exit(1);
        }
        int i = 0;
        const struct ddb_entry *e;
        while ((e = ddb_next(cur))){
                printf("%.*s\n", e->length, e->data);
                ++i;
        }
        ddb_free_cursor(cur);
}

static struct ddb_query_clause *parse_cnf(char **tokens, int num, int *num_clauses)
{
        int k, i, j, t;
        *num_clauses = 1;
        for (i = 0; i < num; i++)
                if (!strcmp(tokens[i], "&"))
                        ++*num_clauses;

        struct ddb_query_term *terms = calloc(num - (*num_clauses - 1),
                sizeof(struct ddb_query_term));
        struct ddb_query_clause *clauses = calloc(*num_clauses,
                sizeof(struct ddb_query_clause));

        clauses[0].terms = terms;
        
        for (t = 0, k = 0, j = 0, i = 0; i < num; i++){
                if (!strcmp(tokens[i], "&")){
                        clauses[j++].num_terms = i - k;
                        clauses[j].terms = &terms[t];
                        k = t;
                        continue;
                }
                if (tokens[i][0] == '-'){
                        terms[t].not = 1;
                        terms[t].key.data = &tokens[i][1];
                        terms[t].key.length = strlen(&tokens[i][1]);
                }else{
                        terms[t].key.data = tokens[i];
                        terms[t].key.length = strlen(tokens[i]);
                }
                ++t;
        }
        clauses[j].num_terms = t - k;
#ifdef DEBUG
        for (i = 0; i < *num_clauses; i++){
                printf("dbg Clause:\n");
                for (j = 0; j < clauses[i].num_terms; j++){
                        if (clauses[i].terms[j].not)
                                printf("dbg NOT ");
                        printf("dbg %.*s\n", clauses[i].terms[j].key.length,
                                clauses[i].terms[j].key.data);
                }
                printf("dbg ---\n");
        }
#endif
        return clauses;
}

static struct ddb *open_discodb(const char *file)
{
        struct stat nfo;
        int fd;
        
        if ((fd = open(file, O_RDONLY)) == -1){
                fprintf(stderr, "Couldn't open discodb %s\n", file);
                exit(1);
        }
        if (fstat(fd, &nfo)){
                fprintf(stderr, "Couldn't get the file size for %s\n", file);
                exit(1);
        }
        
        const char *p = mmap(0, nfo.st_size, PROT_READ, MAP_SHARED, fd, 0);
        if (p == MAP_FAILED){
                fprintf(stderr, "Loading %s failed\n", file);
                exit(1);
        }
        struct ddb *db;
        if (!(db = ddb_new())){
                fprintf(stderr, "Couldn't initialize discodb: Out of memory\n");
                exit(1);
        }
        if (ddb_loads(db, p, nfo.st_size)){
                const char *err;
                ddb_error(db, &err);
                fprintf(stderr, "Invalid discodb in %s: %s\n", file, err);
                exit(1);
        }
        return db;
}

static void usage()
{
        fprintf(stderr, "Usage:\n");
        fprintf(stderr, "query_discodb [discodb] [-keys|-values|-item|-cnf] [query]\n");
        fprintf(stderr, "cnf format example: a b & -c d & e\n");
        exit(1);
}

int main(int argc, char **argv)
{
        if (argc < 3)
                usage();
{'fraction_non_alphanumeric': 0.15029344220464405, 'fraction_numerical': 0.006889512630773156, 'mean_word_length': 1.3309545049063336, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3658947', 'n_tokens_mistral': 2631, 'n_tokens_neox': 2471, 'n_words': 577}