<commit_before>#include <assert.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sqlite3.h>
#include <apr_strings.h>
#include <apr_tables.h>
#include <sys/stat.h>
#include <ctype.h>

#include "set.h"
#include "libcrange.h"
#include "range.h"
char* _join_elements(apr_pool_t* pool, char sep, set* the_set);


const char** functions_provided(libcrange* lr)
{
    static const char* functions[] = {"mem", "cluster", "clusters",
                                      "get_cluster", "get_groups", "group",
                                      "has", "allclusters", 0};
    return functions;
}

#define KEYVALUE_SQL "select key, value from clusters where cluster LIKE ?"
#define HAS_SQL "select cluster from clusters where key LIKE ? and value LIKE ?"
#define GROUPS_SQL "select distinct key from expanded_reverse_clusters where node LIKE ? and cluster LIKE 'GROUPS'"
#define ALLCLUSTER_SQL "select distinct cluster from clusters"
#define CLUSTERS_SQL "select distinct cluster from expanded_reverse_clusters where node LIKE ? and key LIKE 'CLUSTER'"
#define MEM_SQL "select distinct key from expanded_reverse_clusters where cluster LIKE ? and node LIKE ?"
#define EMPTY_STRING ""


sqlite3* _open_db(range_request* rr) 
{
    char * sqlite_db_path;
    sqlite3* db;
    libcrange* lr = range_request_lr(rr);
    int err;
    
    /* open the db */
    if (!(db = libcrange_get_cache(lr, "sqlite:nodes"))) {
        sqlite_db_path = libcrange_getcfg(lr, "sqlitedb");
        if (!sqlite_db_path) sqlite_db_path = DEFAULT_SQLITE_DB;
        
        err = sqlite3_open(sqlite_db_path, &db);<commit_msg>sqlite backend now queries new table<commit_after>#include <assert.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sqlite3.h>
#include <apr_strings.h>
#include <apr_tables.h>
#include <sys/stat.h>
#include <ctype.h>

#include "set.h"
#include "libcrange.h"
#include "range.h"
char* _join_elements(apr_pool_t* pool, char sep, set* the_set);


const char** functions_provided(libcrange* lr)
{
    static const char* functions[] = {"mem", "cluster", "clusters",
                                      "get_cluster", "get_groups", "group",
                                      "has", "allclusters", 0};
    return functions;
}

#define KEYVALUE_SQL "select key, value from clusters where cluster LIKE ?"
#define HAS_SQL "select cluster from clusters_norange where key LIKE ? and value LIKE ?"
#define GROUPS_SQL "select distinct key from expanded_reverse_clusters where node LIKE ? and cluster LIKE 'GROUPS'"
#define ALLCLUSTER_SQL "select distinct cluster from clusters"
#define CLUSTERS_SQL "select distinct cluster from expanded_reverse_clusters where node LIKE ? and key LIKE 'CLUSTER'"
#define MEM_SQL "select distinct key from expanded_reverse_clusters where cluster LIKE ? and node LIKE ?"
#define EMPTY_STRING ""


sqlite3* _open_db(range_request* rr) 
{
    char * sqlite_db_path;
    sqlite3* db;
    libcrange* lr = range_request_lr(rr);
    int err;
    
    /* open the db */
    if (!(db = libcrange_get_cache(lr, "sqlite:nodes"))) {
        sqlite_db_path = libcrange_getcfg(lr, "sqlitedb");
        if (!sqlite_db_path) sqlite_db_path = DEFAULT_SQLITE_DB;
        
        err = sqlite3_open(sqlite_db_path, &db);
{'fraction_non_alphanumeric': 0.11141720232629324, 'fraction_numerical': 0.003060912151821243, 'mean_word_length': 3.7777777777777777, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6406006', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1053, 'n_words': 306}