<commit_before>/* Copyright (C) 2003-2006 <NAME> */

#include "common.h"

#if defined(PASSDB_LDAP) || defined(USERDB_LDAP)

#include "network.h"
#include "ioloop.h"
#include "hash.h"
#include "str.h"
#include "settings.h"
#include "userdb.h"
#include "db-ldap.h"

#include <stddef.h>
#include <stdlib.h>

#define HAVE_LDAP_SASL
#ifdef HAVE_SASL_SASL_H
#  include <sasl/sasl.h>
#elif defined (HAVE_SASL_H)
#  include <sasl.h>
#else
#  undef HAVE_LDAP_SASL
#endif
#if SASL_VERSION_MAJOR < 2
#  undef HAVE_LDAP_SASL
#endif

/* Older versions may require calling ldap_result() twice */
#if LDAP_VENDOR_VERSION <= 20112
#  define OPENLDAP_ASYNC_WORKAROUND
#endif

/* Solaris LDAP library doesn't have LDAP_OPT_SUCCESS */
#ifndef LDAP_OPT_SUCCESS
#  define LDAP_OPT_SUCCESS LDAP_SUCCESS
#endif

#define DEF(type, name) \
	{ type, #name, offsetof(struct ldap_settings, name) }

static struct setting_def setting_defs[] = {
	DEF(SET_STR, hosts),
	DEF(SET_STR, uris),
	DEF(SET_STR, dn),
	DEF(SET_STR, dnpass),
	DEF(SET_BOOL, auth_bind),
	DEF(SET_STR, auth_bind_userdn),
	DEF(SET_BOOL, tls),
	DEF(SET_BOOL, sasl_bind),
	DEF(SET_STR, sasl_mech),
	DEF(SET_STR, sasl_realm),
	DEF(SET_STR, sasl_authz_id),
	DEF(SET_STR, sasl_props),
	DEF(SET_STR, deref),
	DEF(SET_STR, scope),
	DEF(SET_STR, base),
	DEF(SET_INT, ldap_version),
	DEF(SET_STR, user_attrs),
	DEF(SET_STR, user_filter),
	DEF(SET_STR, pass_attrs),
	DEF(SET_STR, pass_filter),
	DEF(SET_STR, default_pass_scheme),
	DEF(SET_STR, user_global_uid),
	DEF(SET_STR, user_global_gid),

	{ 0, NULL, 0 }
};

struct ldap_settings default_ldap_settings = {
	MEMBER(hosts) NULL,
	MEMBER(uris) NULL,
	MEMBER(dn) NULL,
	MEMBER(dnpass) NULL,
	MEMBER(auth_bind) FALSE,
	MEMBER(auth_bind_userdn) NULL,
	MEMBER(tls) FALSE,
	MEMBER(sasl_bind) FALSE,
	MEMBER(sasl_mech) NULL,
	MEMBER(sasl_realm) NULL,
	MEMBER(sasl_authz_id) NULL,
	MEMBER(sasl_props) NULL,
	MEMBER(deref) "never",
	MEMBER(scope) "subtree",
	MEMBER(base) NULL,
	MEMBER(ldap_version) 2,
	MEMBER(user_attrs) "uid,homeDirectory,,,uidNumber,gidNumber",
	MEMBER(user_filter) "(&(objectClass=posixAccount)(uid=%u))",
	MEMBER(pass_attrs) "uid,userPassword",
	MEMBER(pass_filter) "(&(objectClass=posixAccount)(uid=%u))",
	MEMBER(default_pass_scheme) "crypt",
	MEMBER(user_global_uid) "",
	MEMBER(user_global_gid) ""
};

static struct ldap_connection *ldap_connections = NULL;

static void ldap_conn_close(struct ldap_connection *conn, bool flush_requests);

static int deref2str(const char *str)
{
	if (strcasecmp(str, "never") == 0)
		return LDAP_DEREF_NEVER;
	if (strcasecmp(str, "searching") == 0)
		return LDAP_DEREF_SEARCHING;
	if (strcasecmp(str, "finding") == 0)
		return LDAP_DEREF_FINDING;
	if (strcasecmp(str, "always") == 0)
		return LDAP_DEREF_ALWAYS;

	i_fatal("LDAP: Unknown deref option '%s'", str);
}

static int scope2str(const char *str)
{
	if (strcasecmp(str, "base") == 0)
		return LDAP_SCOPE_BASE;
	if (strcasecmp(str, "onelevel") == 0)
		return LDAP_SCOPE_ONELEVEL;
	if (strcasecmp(str, "subtree") == 0)
		return LDAP_SCOPE_SUBTREE;

	i_fatal("LDAP: Unknown scope option '%s'", str);
}

const char *ldap_get_error(struct ldap_connection *conn)
{
	int ret, err;

	ret = ldap_get_option(conn->ld, LDAP_OPT_ERROR_NUMBER, (void *) &err);
	if (ret != LDAP_SUCCESS) {
		i_error("LDAP: Can't get error number: %s",
			ldap_err2string(ret));
		return "??";
	}

	return ldap_err2string(err);
}

void db_ldap_search(struct ldap_connection *conn, struct ldap_request *request,
		    int scope)
{
	int msgid;

	if (!conn->connected) {
		if (!db_ldap_connect(conn)) {
			request->callback(conn, request, NULL);
			return;
		}
	}

	msgid = ldap_search(conn->ld, request->base, scope,
			    request->filter, request->attributes, 0);
	if (msgid == -1) {
		i_error("LDAP: ldap_search() failed (filter %s): %s",
			request->filter, ldap_get_error(conn));
		request->callback(conn, request, NULL);
		return;
	}

	hash_insert(conn->requests, POINTER_CAST(msgid), request);
}

static void ldap_conn_retry_requests(struct ldap_connection *conn)
{
	struct hash_table *old_requests;
        struct hash_iterate_context *iter;
	void *key, *value;

	i_assert(conn->connected);

	if (hash_size(conn->requests) == 0)
		return;

	old_requests = conn->requests;
	conn->requests = hash_create(default_pool, conn->pool, 0, NULL, NULL);

	iter = hash_iterate_init(old_requests);
	while (hash_iterate(iter, &key, &value)) {
		struct ldap_request *request = value;

		i_assert(conn->connected);
		db_ldap_search(conn, request, conn->set.ldap_scope);
	}
	hash_iterate_deinit(iter);
	hash_destroy(old_requests);
}

static void ldap_conn_reconnect(struct ldap_connection *conn)
{
	ldap_conn_close(conn, FALSE);

	if (!db_ldap_connect(conn)) {
		/* failed to reconnect. fail all requests. */
		ldap_conn_close(conn, TRUE);
	}
}

static void ldap_input(void *context)
{
	struct ldap_connection *conn = context;
        struct ldap_request *request;
	struct timeval timeout;
	LDAPMessage *res;
	int ret, msgid;

	while (conn->ld != NULL) {
		memset(&timeout, 0, sizeof(timeout));
		ret = ldap_result(conn->ld, LDAP_RES_ANY, 1, &timeout, &res);
#ifdef OPENLDAP_ASYNC_WORKAROUND
		if (ret == 0) {
			/* try again, there may be another in buffer */
			ret = ldap_result(conn->ld, LDAP_RES_ANY, 1,
					  &timeout, &res);
		}
#endif
		if (ret <= 0) {
			if (ret < 0) {
				i_error("LDAP: ldap_result() failed: %s",
					ldap_get_error(conn));
				ldap_conn_reconnect(conn);
			}
			return;
		}

		msgid = ldap_msgid(res);
		request = hash_lookup(conn->requests, POINTER_CAST(msgid));
		if (request == NULL) {
			i_error("LDAP: Reply with unknown msgid %d",
				msgid);
		} else {
			hash_remove(conn->requests, POINTER_CAST(msgid));
			request->callback(conn, request, res);
		}

		ldap_msgfree(res);
	}
}

#ifdef HAVE_LDAP_SASL
static int
sasl_interact(LDAP *ld __attr_unused__, unsigned flags __attr_unused__,
	      void *defaults, void *interact)
{
	struct ldap_sasl_bind_context *context = defaults;
	sasl_interact_t *in;
	const char *str;

	for (in = interact; in->id != SASL_CB_LIST_END; in++) {
		switch (in->id) {
		case SASL_CB_GETREALM:
			str = context->realm;
			break;
		case SASL_CB_AUTHNAME:
			str = context->authcid;
			break;
		case SASL_CB_USER:
			str = context->authzid;
			break;
		case SASL_CB_PASS:
			str = context->passwd;
			break;
		default:
			str = NULL;
			break;
		}
		if (str != NULL) {
			in->len = strlen(str);
			in->result = str;
		}
		
	}
	return LDAP_SUCCESS;
}
#endif

bool db_ldap_connect(struct ldap_connection *conn)
{
	int ret, fd;

	if (conn->connected)
		return TRUE;

	if (conn->ld == NULL) {
		if (conn->set.uris != NULL) {
#ifdef LDAP_HAVE_INITIALIZE
			if (ldap_initialize(&conn->ld, conn->set.uris) != LDAP_SUCCESS)
				conn->ld = NULL;
#else
			i_fatal("LDAP: Your LDAP library doesn't support "
				"'uris' setting, use 'hosts' instead.");
#endif
		} else
			conn->ld = ldap_init(conn->set.hosts, LDAP_PORT);

		if (conn->ld == NULL)
			i_fatal("LDAP: ldap_init() failed with hosts: %s",
				conn->set.hosts);

		ret = ldap_set_option(conn->ld, LDAP_OPT_DEREF,
				      (void *) &conn->set.ldap_deref);
		if (ret != LDAP_SUCCESS) {
			i_fatal("LDAP: Can't set deref option: %s",
				ldap_err2string(ret));
		}

		ret = ldap_set_option(conn->ld, LDAP_OPT_PROTOCOL_VERSION,
				      (void *) &conn->set.ldap_version);
		if (ret != LDAP_OPT_SUCCESS) {
			i_fatal("LDAP: Can't set protocol version %u: %s",
				conn->set.ldap_version, ldap_err2string(ret));
		}
	}

	if (conn->set.tls) {
#ifdef LDAP_HAVE_START_TLS_S
		ret = ldap_start_tls_s(conn->ld, NULL, NULL);
		if (ret != LDAP_SUCCESS) {
			i_error("LDAP: ldap_start_tls_s() failed: %s",
				ldap_err2string(ret));
			return FALSE;
		}
#else
		i_error("LDAP: Your LDAP library doesn't support TLS");
		return FALSE;
#endif
	}

	/* FIXME: we shouldn't use blocking bind */
	if (conn->set.sasl_bind) {
#ifdef HAVE_LDAP_SASL
		struct ldap_sasl_bind_context context;

		memset(&context, 0, sizeof(context));
		context.authcid = conn->set.dn;
		context.passwd = <PASSWORD>;
		context.realm = conn->set.sasl_realm;
		context.authzid = conn->set.sasl_authz_id;

		ret = ldap_sasl_interactive_bind_s(conn->ld, NULL,
						   conn->set.sasl_mech,
						   NULL, NULL, LDAP_SASL_QUIET,
						   sasl_interact, &context);
#else
		i_fatal("LDAP: sasl_bind=yes but no SASL support compiled in");
#endif
	} else {
		ret = ldap_simple_bind_s(conn->ld, conn->set.dn,
					 conn->set.dnpass);
	}
	if (ret == LDAP_SERVER_DOWN) {
		i_error("LDAP: Can't connect to server: %s",
			conn->set.hosts);
		return FALSE;
	}
	if (ret != LDAP_SUCCESS) {
		i_error("LDAP: binding failed (dn %s): %s",
			conn->set.dn == NULL ? "(none)" : conn->set.dn,
			ldap_get_error(conn));
		return FALSE;
	}

	conn->connected = TRUE;

	/* register LDAP input to ioloop */
	ret = ldap_get_option(conn->ld, LDAP_OPT_DESC, (void *) &fd);
	if (ret != LDAP_SUCCESS) {
		i_fatal("LDAP: Can't get connection fd: %s",
			ldap_err2string(ret));
	}

	net_set_nonblock(fd, TRUE);
	conn->io = io_add(fd, IO_READ, ldap_input, conn);

	/* in case there are requests waiting, retry them */
        ldap_conn_retry_requests(conn);
	return TRUE;
}

static void ldap_conn_close(struct ldap_connection *conn, bool flush_requests)
{
	struct hash_iterate_context *iter;<commit_msg>"Can't connect to server" message's host was wrong if uris setting was used.<commit_after>/* Copyright (C) 2003-2006 <NAME> */

#include "common.h"

#if defined(PASSDB_LDAP) || defined(USERDB_LDAP)

#include "network.h"
#include "ioloop.h"
#include "hash.h"
#include "str.h"
#include "settings.h"
#include "userdb.h"
#include "db-ldap.h"

#include <stddef.h>
#include <stdlib.h>

#define HAVE_LDAP_SASL
#ifdef HAVE_SASL_SASL_H
#  include <sasl/sasl.h>
#elif defined (HAVE_SASL_H)
#  include <sasl.h>
#else
#  undef HAVE_LDAP_SASL
#endif
#if SASL_VERSION_MAJOR < 2
#  undef HAVE_LDAP_SASL
#endif

/* Older versions may require calling ldap_result() twice */
#if LDAP_VENDOR_VERSION <= 20112
#  define OPENLDAP_ASYNC_WORKAROUND
#endif

/* Solaris LDAP library doesn't have LDAP_OPT_SUCCESS */
#ifndef LDAP_OPT_SUCCESS
#  define LDAP_OPT_SUCCESS LDAP_SUCCESS
#endif

#define DEF(type, name) \
	{ type, #name, offsetof(struct ldap_settings, name) }

static struct setting_def setting_defs[] = {
	DEF(SET_STR, hosts),
	DEF(SET_STR, uris),
	DEF(SET_STR, dn),
	DEF(SET_STR, dnpass),
	DEF(SET_BOOL, auth_bind),
	DEF(SET_STR, auth_bind_userdn),
	DEF(SET_BOOL, tls),
	DEF(SET_BOOL, sasl_bind),
	DEF(SET_STR, sasl_mech),
	DEF(SET_STR, sasl_realm),
	DEF(SET_STR, sasl_authz_id),
	DEF(SET_STR, sasl_props),
	DEF(SET_STR, deref),
	DEF(SET_STR, scope),
	DEF(SET_STR, base),
	DEF(SET_INT, ldap_version),
	DEF(SET_STR, user_attrs),
	DEF(SET_STR, user_filter),
	DEF(SET_STR, pass_attrs),
	DEF(SET_STR, pass_filter),
	DEF(SET_STR, default_pass_scheme),
	DEF(SET_STR, user_global_uid),
	DEF(SET_STR, user_global_gid),

	{ 0, NULL, 0 }
};

struct ldap_settings default_ldap_settings = {
	MEMBER(hosts) NULL,
	MEMBER(uris) NULL,
	MEMBER(dn) NULL,
	MEMBER(dnpass) NULL,
	MEMBER(auth_bind) FALSE,
	MEMBER(auth_bind_userdn) NULL,
	MEMBER(tls) FALSE,
	MEMBER(sasl_bind) FALSE,
	MEMBER(sasl_mech) NULL,
	MEMBER(sasl_realm) NULL,
	MEMBER(sasl_authz_id) NULL,
	MEMBER(sasl_props) NULL,
	MEMBER(deref) "never",
	MEMBER(scope) "subtree",
	MEMBER(base) NULL,
	MEMBER(ldap_version) 2,
	MEMBER(user_attrs) "uid,homeDirectory,,,uidNumber,gidNumber",
	MEMBER(user_filter) "(&(objectClass=posixAccount)(uid=%u))",
	MEMBER(pass_attrs) "uid,userPassword",
	MEMBER(pass_filter) "(&(objectClass=posixAccount)(uid=%u))",
	MEMBER(default_pass_scheme) "crypt",
	MEMBER(user_global_uid) "",
	MEMBER(user_global_gid) ""
};

static struct ldap_connection *ldap_connections = NULL;

static void ldap_conn_close(struct ldap_connection *conn, bool flush_requests);

static int deref2str(const char *str)
{
	if (strcasecmp(str, "never") == 0)
		return LDAP_DEREF_NEVER;
	if (strcasecmp(str, "searching") == 0)
		return LDAP_DEREF_SEARCHING;
	if (strcasecmp(str, "finding") == 0)
		return LDAP_DEREF_FINDING;
	if (strcasecmp(str, "always") == 0)
		return LDAP_DEREF_ALWAYS;

	i_fatal("LDAP: Unknown deref option '%s'", str);
}

static int scope2str(const char *str)
{
	if (strcasecmp(str, "base") == 0)
		return LDAP_SCOPE_BASE;
	if (strcasecmp(str, "onelevel") == 0)
		return LDAP_SCOPE_ONELEVEL;
	if (strcasecmp(str, "subtree") == 0)
		return LDAP_SCOPE_SUBTREE;

	i_fatal("LDAP: Unknown scope option '%s'", str);
}

const char *ldap_get_error(struct ldap_connection *conn)
{
	int ret, err;

	ret = ldap_get_option(conn->ld, LDAP_OPT_ERROR_NUMBER, (void *) &err);
	if (ret != LDAP_SUCCESS) {
		i_error("LDAP: Can't get error number: %s",
			ldap_err2string(ret));
		return "??";
	}

	return ldap_err2string(err);
}

void db_ldap_search(struct ldap_connection *conn, struct ldap_request *request,
		    int scope)
{
	int msgid;

	if (!conn->connected) {
		if (!db_ldap_connect(conn)) {
			request->callback(conn, request, NULL);
			return;
		}
	}

	msgid = ldap_search(conn->ld, request->base, scope,
			    request->filter, request->attributes, 0);
	if (msgid == -1) {
		i_error("LDAP: ldap_search() failed (filter %s): %s",
			request->filter, ldap_get_error(conn));
		request->callback(conn, request, NULL);
		return;
	}

	hash_insert(conn->requests, POINTER_CAST(msgid), request);
}

static void ldap_conn_retry_requests(struct ldap_connection *conn)
{
	struct hash_table *old_requests;
        struct hash_iterate_context *iter;
	void *key, *value;

	i_assert(conn->connected);

	if (hash_size(conn->requests) == 0)
		return;

	old_requests = conn->requests;
	conn->requests = hash_create(default_pool, conn->pool, 0, NULL, NULL);

	iter = hash_iterate_init(old_requests);
	while (hash_iterate(iter, &key, &value)) {
		struct ldap_request *request = value;

		i_assert(conn->connected);
		db_ldap_search(conn, request, conn->set.ldap_scope);
	}
	hash_iterate_deinit(iter);
	hash_destroy(old_requests);
}

static void ldap_conn_reconnect(struct ldap_connection *conn)
{
	ldap_conn_close(conn, FALSE);

	if (!db_ldap_connect(conn)) {
		/* failed to reconnect. fail all requests. */
		ldap_conn_close(conn, TRUE);
	}
}

static void ldap_input(void *context)
{
	struct ldap_connection *conn = context;
        struct ldap_request *request;
	struct timeval timeout;
	LDAPMessage *res;
	int ret, msgid;

	while (conn->ld != NULL) {
		memset(&timeout, 0, sizeof(timeout));
		ret = ldap_result(conn->ld, LDAP_RES_ANY, 1, &timeout, &res);
#ifdef OPENLDAP_ASYNC_WORKAROUND
		if (ret == 0) {
			/* try again, there may be another in buffer */
			ret = ldap_result(conn->ld, LDAP_RES_ANY, 1,
					  &timeout, &res);
		}
#endif
		if (ret <= 0) {
			if (ret < 0) {
				i_error("LDAP: ldap_result() failed: %s",
					ldap_get_error(conn));
				ldap_conn_reconnect(conn);
			}
			return;
		}

		msgid = ldap_msgid(res);
		request = hash_lookup(conn->requests, POINTER_CAST(msgid));
		if (request == NULL) {
			i_error("LDAP: Reply with unknown msgid %d",
				msgid);
		} else {
			hash_remove(conn->requests, POINTER_CAST(msgid));
			request->callback(conn, request, res);
		}

		ldap_msgfree(res);
	}
}

#ifdef HAVE_LDAP_SASL
static int
sasl_interact(LDAP *ld __attr_unused__, unsigned flags __attr_unused__,
	      void *defaults, void *interact)
{
	struct ldap_sasl_bind_context *context = defaults;
	sasl_interact_t *in;
	const char *str;

	for (in = interact; in->id != SASL_CB_LIST_END; in++) {
		switch (in->id) {
		case SASL_CB_GETREALM:
			str = context->realm;
			break;
		case SASL_CB_AUTHNAME:
			str = context->authcid;
			break;
		case SASL_CB_USER:
			str = context->authzid;
			break;
		case SASL_CB_PASS:
			str = context->passwd;
			break;
		default:
			str = NULL;
			break;
		}
		if (str != NULL) {
			in->len = strlen(str);
			in->result = str;
		}
		
	}
	return LDAP_SUCCESS;
}
#endif

bool db_ldap_connect(struct ldap_connection *conn)
{
	int ret, fd;

	if (conn->connected)
		return TRUE;

	if (conn->ld == NULL) {
		if (conn->set.uris != NULL) {
#ifdef LDAP_HAVE_INITIALIZE
			if (ldap_initialize(&conn->ld, conn->set.uris) != LDAP_SUCCESS)
				conn->ld = NULL;
#else
			i_fatal("LDAP: Your LDAP library doesn't support "
				"'uris' setting, use 'hosts' instead.");
#endif
		} else
			conn->ld = ldap_init(conn->set.hosts, LDAP_PORT);

		if (conn->ld == NULL)
			i_fatal("LDAP: ldap_init() failed with hosts: %s",
				conn->set.hosts);

		ret = ldap_set_option(conn->ld, LDAP_OPT_DEREF,
				      (void *) &conn->set.ldap_deref);
		if (ret != LDAP_SUCCESS) {
			i_fatal("LDAP: Can't set deref option: %s",
				ldap_err2string(ret));
		}

		ret = ldap_set_option(conn->ld, LDAP_OPT_PROTOCOL_VERSION,
				      (void *) &conn->set.ldap_version);
		if (ret != LDAP_OPT_SUCCESS) {
			i_fatal("LDAP: Can't set protocol version %u: %s",
				conn->set.ldap_version, ldap_err2string(ret));
		}
	}

	if (conn->set.tls) {
#ifdef LDAP_HAVE_START_TLS_S
		ret = ldap_start_tls_s(conn->ld, NULL, NULL);
		if (ret != LDAP_SUCCESS) {
			i_error("LDAP: ldap_start_tls_s() failed: %s",
				ldap_err2string(ret));
			return FALSE;
		}
#else
		i_error("LDAP: Your LDAP library doesn't support TLS");
		return FALSE;
#endif
	}

	/* FIXME: we shouldn't use blocking bind */
	if (conn->set.sasl_bind) {
#ifdef HAVE_LDAP_SASL
		struct ldap_sasl_bind_context context;

		memset(&context, 0, sizeof(context));
		context.authcid = conn->set.dn;
		context.passwd = conn->set.dnpass;
		context.realm = conn->set.sasl_realm;
		context.authzid = conn->set.sasl_authz_id;

		ret = ldap_sasl_interactive_bind_s(conn->ld, NULL,
						   conn->set.sasl_mech,
						   NULL, NULL, LDAP_SASL_QUIET,
						   sasl_interact, &context);
#else
		i_fatal("LDAP: sasl_bind=yes but no SASL support compiled in");
#endif
	} else {
		ret = ldap_simple_bind_s(conn->ld, conn->set.dn,
					 conn->set.dnpass);
	}
	if (ret == LDAP_SERVER_DOWN) {
		i_error("LDAP: Can't connect to server: %s",
			conn->set.uris != NULL ?
			conn->set.uris : conn->set.hosts);
		return FALSE;
	}
	if (ret != LDAP_SUCCESS) {
		i_error("LDAP: binding failed (dn %s): %s",
			conn->set.dn == NULL ? "(none)" : conn->set.dn,
			ldap_get_error(conn));
		return FALSE;
	}

	conn->connected = TRUE;

	/* register LDAP input to ioloop */
	ret = ldap_get_option(conn->ld, LDAP_OPT_DESC, (void *) &fd);
	if (ret != LDAP_SUCCESS) {
		i_fatal("LDAP: Can't get connection fd: %s",
			ldap_err2string(ret));
	}

	net_set_nonblock(fd, TRUE);
	conn->io = io_add(fd, IO_READ, ldap_input, conn);

	/* in case there are requests waiting, retry them */
        ldap_conn_retry_requests(conn);
	return TRUE;
}

static void ldap_conn_close(struct ldap_connection *conn, bool flush_requests)
{
	struct hash_iterate_context *iter;
{'fraction_non_alphanumeric': 0.14305585277123903, 'fraction_numerical': 0.004600898780226835, 'mean_word_length': 4.608460846084609, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7513953', 'n_tokens_mistral': 8132, 'n_tokens_neox': 7460, 'n_words': 1570}