<commit_before>#include <sys/file.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include "rlite/page_btree.h"
#include "rlite/page_list.h"
#include "rlite/page_long.h"
#include "rlite/page_string.h"
#include "rlite/page_skiplist.h"
#include "rlite/page_multi_string.h"
#include "rlite/type_string.h"
#include "rlite/type_zset.h"
#include "rlite/type_hash.h"
#include "rlite.h"
#include "rlite/util.h"
#include "rlite/sha1.h"
#include "rlite/flock.h"
#include "rlite/pubsub.h"
#include "rlite/wal.h"
#ifdef RL_DEBUG
#include <valgrind/valgrind.h>
#endif

#define DEFAULT_READ_PAGES_LEN 16
#define DEFAULT_WRITE_PAGES_LEN 8
#define DEFAULT_PAGE_SIZE 1024
#define HEADER_SIZE 200

int rl_header_serialize(struct rlite *db, void *obj, unsigned char *data);
int rl_has_flag(rlite *db, int flag);

rl_data_type rl_data_type_btree_hash_sha1_long = {
	"rl_data_type_btree_hash_sha1_long",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_sha1_long = {
	"rl_data_type_btree_node_hash_sha1_long",
	rl_btree_node_serialize_hash_sha1_long,
	rl_btree_node_deserialize_hash_sha1_long,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_btree_hash_sha1_double = {
	"rl_data_type_btree_hash_sha1_double",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_sha1_double = {
	"rl_data_type_btree_node_hash_sha1_double",
	rl_btree_node_serialize_hash_sha1_double,
	rl_btree_node_deserialize_hash_sha1_double,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_btree_hash_sha1_key = {
	"rl_data_type_btree_hash_sha1_key",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_sha1_key = {
	"rl_data_type_btree_node_hash_sha1_key",
	rl_btree_node_serialize_hash_sha1_key,
	rl_btree_node_deserialize_hash_sha1_key,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_btree_hash_sha1_hashkey = {
	"rl_data_type_btree_hash_sha1_hashkey",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_sha1_hashkey = {
	"rl_data_type_btree_node_hash_sha1_hashkey",
	rl_btree_node_serialize_hash_sha1_hashkey,
	rl_btree_node_deserialize_hash_sha1_hashkey,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_header = {
	"rl_data_type_header",
	rl_header_serialize,
	rl_header_deserialize,
	NULL
};
rl_data_type rl_data_type_btree_hash_long_long = {
	"rl_data_type_btree_hash_long_long",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_long_long = {
	"rl_data_type_btree_node_hash_long_long",
	rl_btree_node_serialize_hash_long_long,
	rl_btree_node_deserialize_hash_long_long,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_btree_hash_double_long = {
	"rl_data_type_btree_hash_double_long",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_list_long = {
	"rl_data_type_list_long",
	rl_list_serialize,
	rl_list_deserialize,
	rl_list_destroy,
};
rl_data_type rl_data_type_list_node_long = {
	"rl_data_type_list_node_long",
	rl_list_node_serialize_long,
	rl_list_node_deserialize_long,
	rl_list_node_destroy,
};
rl_data_type rl_data_type_string = {
	"rl_data_type_string",
	rl_string_serialize,
	rl_string_deserialize,
	rl_string_destroy,
};

rl_data_type rl_data_type_skiplist = {
	"rl_data_type_skiplist",
	rl_skiplist_serialize,
	rl_skiplist_deserialize,
	rl_skiplist_destroy,
};

rl_data_type rl_data_type_skiplist_node = {
	"rl_data_type_skiplist_node",
	rl_skiplist_node_serialize,
	rl_skiplist_node_deserialize,
	rl_skiplist_node_destroy,
};
rl_data_type rl_data_type_long = {
	"rl_data_type_long",
	rl_long_serialize,
	rl_long_deserialize,
	rl_long_destroy,
};

rl_data_type rl_data_type_skiplist_node;

static const unsigned char *identifier = (unsigned char *)"rlite0.0";

static int file_driver_fp(rlite *db)
{
	int retval = RL_OK;
	rl_file_driver *driver = db->driver;
	if (driver->fp == NULL) {
		char *mode;
		if (access(driver->filename, F_OK) == 0) {
			mode = "r+";
		}
		else {
			if ((driver->mode & RLITE_OPEN_READWRITE) == 0) {
				fprintf(stderr, "Opening unexisting file in readonly mode\n");
				retval = RL_INVALID_PARAMETERS;
				goto cleanup;
			}
			mode = "w+";
		}
		driver->fp = fopen(driver->filename, mode);
		if (driver->fp == NULL) {
			fprintf(stderr, "Cannot open file %s, errno %d, mode %s\n", driver->filename, errno, mode);
			perror(NULL);
			retval = RL_UNEXPECTED;
			goto cleanup;
		}
		retval = rl_flock(driver->fp, (driver->mode & RLITE_OPEN_READWRITE) ? RLITE_FLOCK_EX : RLITE_FLOCK_SH);
		if (retval != RL_OK) {
			fclose(driver->fp);
			driver->fp = NULL;
			goto cleanup;
		}
	}
cleanup:
	return retval;
}

int rl_header_serialize(struct rlite *db, void *UNUSED(obj), unsigned char *data)
{
	int identifier_len = strlen((char *)identifier);
	memcpy(data, identifier, identifier_len);
	put_4bytes(&data[identifier_len], db->page_size);
	put_4bytes(&data[identifier_len + 4], db->next_empty_page);
	put_4bytes(&data[identifier_len + 8], db->number_of_pages);
	put_4bytes(&data[identifier_len + 12], db->number_of_databases);
	long i, pos = identifier_len + 16;
	for (i = 0; i < db->number_of_databases + RLITE_INTERNAL_DB_COUNT; i++) {
		if (db->databases[i] != 0) {
			put_4bytes(&data[pos], db->databases[i]);
		}
		pos += 4;
	}
	return RL_OK;
}

int rl_header_deserialize(struct rlite *db, void **UNUSED(obj), void *UNUSED(context), unsigned char *data)
{
	int retval = RL_OK;
	int identifier_len = strlen((char *)identifier);
	if (memcmp(data, identifier, identifier_len) != 0) {
		fprintf(stderr, "Unexpected header, expecting %s\n", identifier);
		return RL_INVALID_STATE;
	}
	db->page_size = get_4bytes(&data[identifier_len]);
	db->initial_next_empty_page =
	db->next_empty_page = get_4bytes(&data[identifier_len + 4]);
	db->initial_number_of_pages =
	db->number_of_pages = get_4bytes(&data[identifier_len + 8]);
	db->initial_number_of_databases =
	db->number_of_databases = get_4bytes(&data[identifier_len + 12]);
	rl_free(db->databases);
	rl_free(db->initial_databases);
	RL_MALLOC(db->databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));
	RL_MALLOC(db->initial_databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));

	long i, pos = identifier_len + 16;
	for (i = 0; i < db->number_of_databases + RLITE_INTERNAL_DB_COUNT; i++) {
		db->initial_databases[i] =
		db->databases[i] = get_4bytes(&data[pos]);
		pos += 4;
	}
cleanup:
	return retval;
}

int rl_ensure_pages(rlite *db)
{
	int retval = RL_OK;
	void *tmp;
	if (rl_has_flag(db, RLITE_OPEN_READWRITE)) {
		if (db->write_pages_len == db->write_pages_alloc) {
			RL_REALLOC(db->write_pages, sizeof(rl_page *) * db->write_pages_alloc * 2)
			db->write_pages_alloc *= 2;
		}
	}
	if (db->read_pages_len == db->read_pages_alloc) {
		RL_REALLOC(db->read_pages, sizeof(rl_page *) * db->read_pages_alloc * 2)
		db->read_pages_alloc *= 2;
	}
cleanup:
	return retval;
}

int rl_open(const char *filename, rlite **_db, int flags)
{
	int retval = RL_OK;
	rlite *db;
	RL_MALLOC(db, sizeof(*db));

	db->subscriber_lock_filename = NULL;
	db->subscriber_id = NULL;
	db->databases = NULL;
	db->initial_databases = NULL;
	db->selected_database = 0;
	db->selected_internal = RLITE_INTERNAL_DB_NO;
	db->page_size = DEFAULT_PAGE_SIZE;
	db->read_pages = db->write_pages = NULL;
	db->read_pages_alloc = db->read_pages_len = db->write_pages_len = db->write_pages_alloc = 0;
	db->initial_number_of_pages = db->number_of_pages = 0;
	db->initial_number_of_databases =
	db->number_of_databases = 0;
	db->driver = NULL;
	db->driver_type = -1;

	RL_MALLOC(db->read_pages, sizeof(rl_page *) * DEFAULT_READ_PAGES_LEN)
	db->read_pages_len = 0;
	db->read_pages_alloc = DEFAULT_READ_PAGES_LEN;
	db->write_pages_len = 0;
	if ((flags & RLITE_OPEN_READWRITE) > 0) {
		db->write_pages_alloc = DEFAULT_WRITE_PAGES_LEN;
		RL_MALLOC(db->write_pages, sizeof(rl_page *) * DEFAULT_WRITE_PAGES_LEN);
	}
	else {
		db->write_pages_alloc = 0;
	}

	if (memcmp(filename, ":memory:", 9) == 0) {
		rl_memory_driver *driver;
		RL_MALLOC(driver, sizeof(*driver));
		db->driver_type = RL_MEMORY_DRIVER;
		db->driver = driver;
		driver->data = NULL;
		driver->datalen = 0;
	}
	else {
		if ((flags & RLITE_OPEN_CREATE) == 0) {
			int _access_flags;
			if ((flags & RLITE_OPEN_READWRITE) != 0) {
				_access_flags = W_OK;
			}
			else {
				_access_flags = R_OK | F_OK;
			}
			if (access(filename, _access_flags) != 0) {
				retval = RL_INVALID_PARAMETERS;
				goto cleanup;
			}
		}

		rl_file_driver *driver;
		RL_MALLOC(driver, sizeof(*driver));
		driver->fp = NULL;
		driver->filename = rl_malloc(sizeof(char) * (strlen(filename) + 1));
		if (!driver->filename) {
			rl_free(driver);
			retval = RL_OUT_OF_MEMORY;
			goto cleanup;
		}
		strcpy(driver->filename, filename);
		driver->mode = flags;
		db->driver = driver;
		db->driver_type = RL_FILE_DRIVER;
	}

	RL_CALL(rl_read_header, RL_OK, db);

	*_db = db;
cleanup:
	if (retval != RL_OK && db != NULL) {
		rl_close(db);
	}
	return retval;
}

int rl_refresh(rlite *db)
{
	int retval = RL_OK;
	if (db->driver_type == RL_FILE_DRIVER) {
		RL_CALL(rl_discard, RL_OK, db);
		RL_CALL(rl_read_header, RL_OK, db);
	}
cleanup:
	return retval;
}

int rl_close(rlite *db)
{
	if (!db) {
		return RL_OK;
	}

	rl_unsubscribe_all(db);
	// discard before removing the driver, since we need to release locks
	rl_discard(db);
	if (db->driver_type == RL_FILE_DRIVER) {<commit_msg>Only unsubscribe when using a file driver<commit_after>#include <sys/file.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include "rlite/page_btree.h"
#include "rlite/page_list.h"
#include "rlite/page_long.h"
#include "rlite/page_string.h"
#include "rlite/page_skiplist.h"
#include "rlite/page_multi_string.h"
#include "rlite/type_string.h"
#include "rlite/type_zset.h"
#include "rlite/type_hash.h"
#include "rlite.h"
#include "rlite/util.h"
#include "rlite/sha1.h"
#include "rlite/flock.h"
#include "rlite/pubsub.h"
#include "rlite/wal.h"
#ifdef RL_DEBUG
#include <valgrind/valgrind.h>
#endif

#define DEFAULT_READ_PAGES_LEN 16
#define DEFAULT_WRITE_PAGES_LEN 8
#define DEFAULT_PAGE_SIZE 1024
#define HEADER_SIZE 200

int rl_header_serialize(struct rlite *db, void *obj, unsigned char *data);
int rl_has_flag(rlite *db, int flag);

rl_data_type rl_data_type_btree_hash_sha1_long = {
	"rl_data_type_btree_hash_sha1_long",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_sha1_long = {
	"rl_data_type_btree_node_hash_sha1_long",
	rl_btree_node_serialize_hash_sha1_long,
	rl_btree_node_deserialize_hash_sha1_long,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_btree_hash_sha1_double = {
	"rl_data_type_btree_hash_sha1_double",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_sha1_double = {
	"rl_data_type_btree_node_hash_sha1_double",
	rl_btree_node_serialize_hash_sha1_double,
	rl_btree_node_deserialize_hash_sha1_double,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_btree_hash_sha1_key = {
	"rl_data_type_btree_hash_sha1_key",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_sha1_key = {
	"rl_data_type_btree_node_hash_sha1_key",
	rl_btree_node_serialize_hash_sha1_key,
	rl_btree_node_deserialize_hash_sha1_key,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_btree_hash_sha1_hashkey = {
	"rl_data_type_btree_hash_sha1_hashkey",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_sha1_hashkey = {
	"rl_data_type_btree_node_hash_sha1_hashkey",
	rl_btree_node_serialize_hash_sha1_hashkey,
	rl_btree_node_deserialize_hash_sha1_hashkey,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_header = {
	"rl_data_type_header",
	rl_header_serialize,
	rl_header_deserialize,
	NULL
};
rl_data_type rl_data_type_btree_hash_long_long = {
	"rl_data_type_btree_hash_long_long",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_btree_node_hash_long_long = {
	"rl_data_type_btree_node_hash_long_long",
	rl_btree_node_serialize_hash_long_long,
	rl_btree_node_deserialize_hash_long_long,
	rl_btree_node_destroy,
};
rl_data_type rl_data_type_btree_hash_double_long = {
	"rl_data_type_btree_hash_double_long",
	rl_btree_serialize,
	rl_btree_deserialize,
	rl_btree_destroy,
};
rl_data_type rl_data_type_list_long = {
	"rl_data_type_list_long",
	rl_list_serialize,
	rl_list_deserialize,
	rl_list_destroy,
};
rl_data_type rl_data_type_list_node_long = {
	"rl_data_type_list_node_long",
	rl_list_node_serialize_long,
	rl_list_node_deserialize_long,
	rl_list_node_destroy,
};
rl_data_type rl_data_type_string = {
	"rl_data_type_string",
	rl_string_serialize,
	rl_string_deserialize,
	rl_string_destroy,
};

rl_data_type rl_data_type_skiplist = {
	"rl_data_type_skiplist",
	rl_skiplist_serialize,
	rl_skiplist_deserialize,
	rl_skiplist_destroy,
};

rl_data_type rl_data_type_skiplist_node = {
	"rl_data_type_skiplist_node",
	rl_skiplist_node_serialize,
	rl_skiplist_node_deserialize,
	rl_skiplist_node_destroy,
};
rl_data_type rl_data_type_long = {
	"rl_data_type_long",
	rl_long_serialize,
	rl_long_deserialize,
	rl_long_destroy,
};

rl_data_type rl_data_type_skiplist_node;

static const unsigned char *identifier = (unsigned char *)"rlite0.0";

static int file_driver_fp(rlite *db)
{
	int retval = RL_OK;
	rl_file_driver *driver = db->driver;
	if (driver->fp == NULL) {
		char *mode;
		if (access(driver->filename, F_OK) == 0) {
			mode = "r+";
		}
		else {
			if ((driver->mode & RLITE_OPEN_READWRITE) == 0) {
				fprintf(stderr, "Opening unexisting file in readonly mode\n");
				retval = RL_INVALID_PARAMETERS;
				goto cleanup;
			}
			mode = "w+";
		}
		driver->fp = fopen(driver->filename, mode);
		if (driver->fp == NULL) {
			fprintf(stderr, "Cannot open file %s, errno %d, mode %s\n", driver->filename, errno, mode);
			perror(NULL);
			retval = RL_UNEXPECTED;
			goto cleanup;
		}
		retval = rl_flock(driver->fp, (driver->mode & RLITE_OPEN_READWRITE) ? RLITE_FLOCK_EX : RLITE_FLOCK_SH);
		if (retval != RL_OK) {
			fclose(driver->fp);
			driver->fp = NULL;
			goto cleanup;
		}
	}
cleanup:
	return retval;
}

int rl_header_serialize(struct rlite *db, void *UNUSED(obj), unsigned char *data)
{
	int identifier_len = strlen((char *)identifier);
	memcpy(data, identifier, identifier_len);
	put_4bytes(&data[identifier_len], db->page_size);
	put_4bytes(&data[identifier_len + 4], db->next_empty_page);
	put_4bytes(&data[identifier_len + 8], db->number_of_pages);
	put_4bytes(&data[identifier_len + 12], db->number_of_databases);
	long i, pos = identifier_len + 16;
	for (i = 0; i < db->number_of_databases + RLITE_INTERNAL_DB_COUNT; i++) {
		if (db->databases[i] != 0) {
			put_4bytes(&data[pos], db->databases[i]);
		}
		pos += 4;
	}
	return RL_OK;
}

int rl_header_deserialize(struct rlite *db, void **UNUSED(obj), void *UNUSED(context), unsigned char *data)
{
	int retval = RL_OK;
	int identifier_len = strlen((char *)identifier);
	if (memcmp(data, identifier, identifier_len) != 0) {
		fprintf(stderr, "Unexpected header, expecting %s\n", identifier);
		return RL_INVALID_STATE;
	}
	db->page_size = get_4bytes(&data[identifier_len]);
	db->initial_next_empty_page =
	db->next_empty_page = get_4bytes(&data[identifier_len + 4]);
	db->initial_number_of_pages =
	db->number_of_pages = get_4bytes(&data[identifier_len + 8]);
	db->initial_number_of_databases =
	db->number_of_databases = get_4bytes(&data[identifier_len + 12]);
	rl_free(db->databases);
	rl_free(db->initial_databases);
	RL_MALLOC(db->databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));
	RL_MALLOC(db->initial_databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));

	long i, pos = identifier_len + 16;
	for (i = 0; i < db->number_of_databases + RLITE_INTERNAL_DB_COUNT; i++) {
		db->initial_databases[i] =
		db->databases[i] = get_4bytes(&data[pos]);
		pos += 4;
	}
cleanup:
	return retval;
}

int rl_ensure_pages(rlite *db)
{
	int retval = RL_OK;
	void *tmp;
	if (rl_has_flag(db, RLITE_OPEN_READWRITE)) {
		if (db->write_pages_len == db->write_pages_alloc) {
			RL_REALLOC(db->write_pages, sizeof(rl_page *) * db->write_pages_alloc * 2)
			db->write_pages_alloc *= 2;
		}
	}
	if (db->read_pages_len == db->read_pages_alloc) {
		RL_REALLOC(db->read_pages, sizeof(rl_page *) * db->read_pages_alloc * 2)
		db->read_pages_alloc *= 2;
	}
cleanup:
	return retval;
}

int rl_open(const char *filename, rlite **_db, int flags)
{
	int retval = RL_OK;
	rlite *db;
	RL_MALLOC(db, sizeof(*db));

	db->subscriber_lock_filename = NULL;
	db->subscriber_id = NULL;
	db->databases = NULL;
	db->initial_databases = NULL;
	db->selected_database = 0;
	db->selected_internal = RLITE_INTERNAL_DB_NO;
	db->page_size = DEFAULT_PAGE_SIZE;
	db->read_pages = db->write_pages = NULL;
	db->read_pages_alloc = db->read_pages_len = db->write_pages_len = db->write_pages_alloc = 0;
	db->initial_number_of_pages = db->number_of_pages = 0;
	db->initial_number_of_databases =
	db->number_of_databases = 0;
	db->driver = NULL;
	db->driver_type = -1;

	RL_MALLOC(db->read_pages, sizeof(rl_page *) * DEFAULT_READ_PAGES_LEN)
	db->read_pages_len = 0;
	db->read_pages_alloc = DEFAULT_READ_PAGES_LEN;
	db->write_pages_len = 0;
	if ((flags & RLITE_OPEN_READWRITE) > 0) {
		db->write_pages_alloc = DEFAULT_WRITE_PAGES_LEN;
		RL_MALLOC(db->write_pages, sizeof(rl_page *) * DEFAULT_WRITE_PAGES_LEN);
	}
	else {
		db->write_pages_alloc = 0;
	}

	if (memcmp(filename, ":memory:", 9) == 0) {
		rl_memory_driver *driver;
		RL_MALLOC(driver, sizeof(*driver));
		db->driver_type = RL_MEMORY_DRIVER;
		db->driver = driver;
		driver->data = NULL;
		driver->datalen = 0;
	}
	else {
		if ((flags & RLITE_OPEN_CREATE) == 0) {
			int _access_flags;
			if ((flags & RLITE_OPEN_READWRITE) != 0) {
				_access_flags = W_OK;
			}
			else {
				_access_flags = R_OK | F_OK;
			}
			if (access(filename, _access_flags) != 0) {
				retval = RL_INVALID_PARAMETERS;
				goto cleanup;
			}
		}

		rl_file_driver *driver;
		RL_MALLOC(driver, sizeof(*driver));
		driver->fp = NULL;
		driver->filename = rl_malloc(sizeof(char) * (strlen(filename) + 1));
		if (!driver->filename) {
			rl_free(driver);
			retval = RL_OUT_OF_MEMORY;
			goto cleanup;
		}
		strcpy(driver->filename, filename);
		driver->mode = flags;
		db->driver = driver;
		db->driver_type = RL_FILE_DRIVER;
	}

	RL_CALL(rl_read_header, RL_OK, db);

	*_db = db;
cleanup:
	if (retval != RL_OK && db != NULL) {
		rl_close(db);
	}
	return retval;
}

int rl_refresh(rlite *db)
{
	int retval = RL_OK;
	if (db->driver_type == RL_FILE_DRIVER) {
		RL_CALL(rl_discard, RL_OK, db);
		RL_CALL(rl_read_header, RL_OK, db);
	}
cleanup:
	return retval;
}

int rl_close(rlite *db)
{
	if (!db) {
		return RL_OK;
	}

	if (db->driver_type == RL_FILE_DRIVER) {
		rl_unsubscribe_all(db);
	}
	// discard before removing the driver, since we need to release locks
	rl_discard(db);
	if (db->driver_type == RL_FILE_DRIVER) {
{'fraction_non_alphanumeric': 0.12026633374947981, 'fraction_numerical': 0.009051186017478152, 'mean_word_length': 6.008749544294568, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 188, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1294236', 'n_tokens_mistral': 8335, 'n_tokens_neox': 7960, 'n_words': 1241}