<commit_before>/*
 * libkmod - interface to kernel module operations
 *
 * Copyright (C) 2011-2013  ProFUSION embedded systems
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <ctype.h>
#include <dirent.h>
#include <errno.h>
#include <fnmatch.h>
#include <inttypes.h>
#include <limits.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <sys/wait.h>
#ifdef HAVE_LINUX_MODULE_H
#include <linux/module.h>
#endif

#include <shared/util.h>

#include "libkmod.h"
#include "libkmod-internal.h"

/**
 * SECTION:libkmod-module
 * @short_description: operate on kernel modules
 */

enum kmod_module_builtin {
    KMOD_MODULE_BUILTIN_UNKNOWN,
    KMOD_MODULE_BUILTIN_NO,
    KMOD_MODULE_BUILTIN_YES,
};

/**
 * kmod_module:
 *
 * Opaque object representing a module.
 */
struct kmod_module {
	struct kmod_ctx *ctx;
	char *hashkey;
	char *name;
	char *path;
	struct kmod_list *dep;
	char *options;
	const char *install_commands;	/* owned by kmod_config */
	const char *remove_commands;	/* owned by kmod_config */
	char *alias; /* only set if this module was created from an alias */
	struct kmod_file *file;
	int n_dep;
	int refcount;
	struct {
		bool dep : 1;
		bool options : 1;
		bool install_commands : 1;
		bool remove_commands : 1;
	} init;

	/*
	 * mark if module is builtin, i.e. it's present on modules.builtin
	 * file. This is set as soon as it is needed or as soon as we know
	 * about it, i.e. the module was created from builtin lookup.
	 */
	enum kmod_module_builtin builtin;

	/*
	 * private field used by kmod_module_get_probe_list() to detect
	 * dependency loops
	 */
	bool visited : 1;

	/*
	 * set by kmod_module_get_probe_list: indicates for probe_insert()
	 * whether the module's command and softdep should be ignored
	 */
	bool ignorecmd : 1;

	/*
	 * set by kmod_module_get_probe_list: indicates whether this is the
	 * module the user asked for or its dependency, or whether this
	 * is a softdep only
	 */
	bool required : 1;
};

static inline const char *path_join(const char *path, size_t prefixlen,
							char buf[PATH_MAX])
{
	size_t pathlen;

	if (path[0] == '/')
		return path;

	pathlen = strlen(path);
	if (prefixlen + pathlen + 1 >= PATH_MAX)
		return NULL;

	memcpy(buf + prefixlen, path, pathlen + 1);
	return buf;
}

static inline bool module_is_inkernel(struct kmod_module *mod)
{
	int state = kmod_module_get_initstate(mod);

	if (state == KMOD_MODULE_LIVE ||
			state == KMOD_MODULE_BUILTIN)
		return true;

	return false;
}

int kmod_module_parse_depline(struct kmod_module *mod, char *line)
{
	struct kmod_ctx *ctx = mod->ctx;
	struct kmod_list *list = NULL;
	const char *dirname;
	char buf[PATH_MAX];
	char *p, *saveptr;
	int err = 0, n = 0;
	size_t dirnamelen;

	if (mod->init.dep)
		return mod->n_dep;
	assert(mod->dep == NULL);
	mod->init.dep = true;

	p = strchr(line, ':');
	if (p == NULL)
		return 0;

	*p = '\0';
	dirname = kmod_get_dirname(mod->ctx);
	dirnamelen = strlen(dirname);
	if (dirnamelen + 2 >= PATH_MAX)
		return 0;

	memcpy(buf, dirname, dirnamelen);
	buf[dirnamelen] = '/';
	dirnamelen++;
	buf[dirnamelen] = '\0';

	if (mod->path == NULL) {
		const char *str = path_join(line, dirnamelen, buf);
		if (str == NULL)
			return 0;
		mod->path = strdup(str);
		if (mod->path == NULL)
			return 0;
	}

	p++;
	for (p = strtok_r(p, " \t", &saveptr); p != NULL;
					p = strtok_r(NULL, " \t", &saveptr)) {
		struct kmod_module *depmod = NULL;
		const char *path;

		path = path_join(p, dirnamelen, buf);
		if (path == NULL) {
			ERR(ctx, "could not join path '%s' and '%s'.\n",
			    dirname, p);
			goto fail;
		}

		err = kmod_module_new_from_path(ctx, path, &depmod);
		if (err < 0) {
			ERR(ctx, "ctx=%p path=%s error=%s\n",
						ctx, path, strerror(-err));
			goto fail;
		}

		DBG(ctx, "add dep: %s\n", path);

		list = kmod_list_prepend(list, depmod);
		n++;
	}

	DBG(ctx, "%d dependencies for %s\n", n, mod->name);

	mod->dep = list;
	mod->n_dep = n;
	return n;

fail:
	kmod_module_unref_list(list);
	mod->init.dep = false;
	return err;
}

void kmod_module_set_visited(struct kmod_module *mod, bool visited)
{
	mod->visited = visited;
}

void kmod_module_set_builtin(struct kmod_module *mod, bool builtin)
{
	mod->builtin =
		builtin ? KMOD_MODULE_BUILTIN_YES : KMOD_MODULE_BUILTIN_NO;
}

void kmod_module_set_required(struct kmod_module *mod, bool required)
{
	mod->required = required;
}

bool kmod_module_is_builtin(struct kmod_module *mod)
{
	if (mod->builtin == KMOD_MODULE_BUILTIN_UNKNOWN) {
		kmod_module_set_builtin(mod,
					kmod_lookup_alias_is_builtin(mod->ctx, mod->name));
	}

	return mod->builtin == KMOD_MODULE_BUILTIN_YES;
}
/*
 * Memory layout with alias:
 *
 * struct kmod_module {
 *        hashkey -----.
 *        alias -----. |
 *        name ----. | |
 * }               | | |
 * name <----------' | |
 * alias <-----------' |
 * name\alias <--------'
 *
 * Memory layout without alias:
 *
 * struct kmod_module {
 *        hashkey ---.
 *        alias -----|----> NULL
 *        name ----. |
 * }               | |
 * name <----------'-'
 *
 * @key is "name\alias" or "name" (in which case alias == NULL)
 */
static int kmod_module_new(struct kmod_ctx *ctx, const char *key,
				const char *name, size_t namelen,
				const char *alias, size_t aliaslen,
				struct kmod_module **mod)
{
	struct kmod_module *m;
	size_t keylen;

	m = kmod_pool_get_module(ctx, key);
	if (m != NULL) {
		*mod = kmod_module_ref(m);
		return 0;
	}

	if (alias == NULL)
		keylen = namelen;
	else
		keylen = namelen + aliaslen + 1;

	m = malloc(sizeof(*m) + (alias == NULL ? 1 : 2) * (keylen + 1));
	if (m == NULL)
		return -ENOMEM;

	memset(m, 0, sizeof(*m));

	m->ctx = kmod_ref(ctx);
	m->name = (char *)m + sizeof(*m);
	memcpy(m->name, key, keylen + 1);
	if (alias == NULL) {
		m->hashkey = m->name;
		m->alias = NULL;
	} else {
		m->name[namelen] = '\0';
		m->alias = m->name + namelen + 1;
		m->hashkey = m->name + keylen + 1;
		memcpy(m->hashkey, key, keylen + 1);
	}

	m->refcount = 1;
	kmod_pool_add_module(ctx, m, m->hashkey);
	*mod = m;

	return 0;
}

/**
 * kmod_module_new_from_name:
 * @ctx: kmod library context
 * @name: name of the module
 * @mod: where to save the created struct kmod_module
 *
 * Create a new struct kmod_module using the module name. @name can not be an
 * alias, file name or anything else; it must be a module name. There's no
 * check if the module exists in the system.
 *
 * This function is also used internally by many others that return a new
 * struct kmod_module or a new list of modules.
 *
 * The initial refcount is 1, and needs to be decremented to release the
 * resources of the kmod_module. Since libkmod keeps track of all
 * kmod_modules created, they are all released upon @ctx destruction too. Do
 * not unref @ctx before all the desired operations with the returned
 * kmod_module are done.
 *
 * Returns: 0 on success or < 0 otherwise. It fails if name is not a valid
 * module name or if memory allocation failed.
 */
KMOD_EXPORT int kmod_module_new_from_name(struct kmod_ctx *ctx,
						const char *name,
						struct kmod_module **mod)
{
	size_t namelen;
	char name_norm[PATH_MAX];

	if (ctx == NULL || name == NULL || mod == NULL)
		return -ENOENT;

	modname_normalize(name, name_norm, &namelen);

	return kmod_module_new(ctx, name_norm, name_norm, namelen, NULL, 0, mod);
}

int kmod_module_new_from_alias(struct kmod_ctx *ctx, const char *alias,
				const char *name, struct kmod_module **mod)
{
	int err;
	char key[PATH_MAX];
	size_t namelen = strlen(name);
	size_t aliaslen = strlen(alias);

	if (namelen + aliaslen + 2 > PATH_MAX)
		return -ENAMETOOLONG;

	memcpy(key, name, namelen);
	memcpy(key + namelen + 1, alias, aliaslen + 1);
	key[namelen] = '\\';

	err = kmod_module_new(ctx, key, name, namelen, alias, aliaslen, mod);
	if (err < 0)
		return err;

	return 0;
}

/**
 * kmod_module_new_from_path:
 * @ctx: kmod library context
 * @path: path where to find the given module
 * @mod: where to save the created struct kmod_module
 *
 * Create a new struct kmod_module using the module path. @path must be an
 * existent file with in the filesystem and must be accessible to libkmod.
 *
 * The initial refcount is 1, and needs to be decremented to release the
 * resources of the kmod_module. Since libkmod keeps track of all
 * kmod_modules created, they are all released upon @ctx destruction too. Do
 * not unref @ctx before all the desired operations with the returned
 * kmod_module are done.
 *
 * If @path is relative, it's treated as relative to the current working
 * directory. Otherwise, give an absolute path.
 *
 * Returns: 0 on success or < 0 otherwise. It fails if file does not exist, if
 * it's not a valid file for a kmod_module or if memory allocation failed.
 */
KMOD_EXPORT int kmod_module_new_from_path(struct kmod_ctx *ctx,
						const char *path,
						struct kmod_module **mod)
{
	struct kmod_module *m;
	int err;
	struct stat st;
	char name[PATH_MAX];
	char *abspath;
	size_t namelen;

	if (ctx == NULL || path == NULL || mod == NULL)
		return -ENOENT;

	abspath = path_make_absolute_cwd(path);
	if (abspath == NULL) {
		DBG(ctx, "no absolute path for %s\n", path);
		return -ENOMEM;
	}

	err = stat(abspath, &st);
	if (err < 0) {
		err = -errno;
		DBG(ctx, "stat %s: %s\n", path, strerror(errno));
		free(abspath);
		return err;
	}

	if (path_to_modname(path, name, &namelen) == NULL) {
		DBG(ctx, "could not get modname from path %s\n", path);
		free(abspath);
		return -ENOENT;
	}

	m = kmod_pool_get_module(ctx, name);
	if (m != NULL) {
		if (m->path == NULL)
			m->path = abspath;
		else if (streq(m->path, abspath))
			free(abspath);
		else {
			ERR(ctx, "kmod_module '%s' already exists with different path: new-path='%s' old-path='%s'\n",
							name, abspath, m->path);
			free(abspath);
			return -EEXIST;
		}

		*mod = kmod_module_ref(m);
		return 0;
	}

	err = kmod_module_new(ctx, name, name, namelen, NULL, 0, &m);
	if (err < 0) {
		free(abspath);
		return err;
	}

	m->path = abspath;
	*mod = m;

	return 0;
}

/**
 * kmod_module_unref:
 * @mod: kmod module
 *
 * Drop a reference of the kmod module. If the refcount reaches zero, its
 * resources are released.
 *
 * Returns: NULL if @mod is NULL or if the module was released. Otherwise it
 * returns the passed @mod with its refcount decremented.
 */
KMOD_EXPORT struct kmod_module *kmod_module_unref(struct kmod_module *mod)
{
	if (mod == NULL)
		return NULL;

	if (--mod->refcount > 0)
		return mod;

	DBG(mod->ctx, "kmod_module %p released\n", mod);

	kmod_pool_del_module(mod->ctx, mod, mod->hashkey);
	kmod_module_unref_list(mod->dep);

	if (mod->file)
		kmod_file_unref(mod->file);

	kmod_unref(mod->ctx);
	free(mod->options);
	free(mod->path);
	free(mod);
	return NULL;
}

/**
 * kmod_module_ref:
 * @mod: kmod module
 *
 * Take a reference of the kmod module, incrementing its refcount.
 *
 * Returns: the passed @module with its refcount incremented.
 */
KMOD_EXPORT struct kmod_module *kmod_module_ref(struct kmod_module *mod)
{
	if (mod == NULL)
		return NULL;

	mod->refcount++;

	return mod;
}

#define CHECK_ERR_AND_FINISH(_err, _label_err, _list, label_finish)	\
	do {								\
		if ((_err) < 0)						\
			goto _label_err;				\
		if (*(_list) != NULL)					\
			goto finish;					\
	} while (0)

/**
 * kmod_module_new_from_lookup:
 * @ctx: kmod library context
 * @given_alias: alias to look for
 * @list: an empty list where to save the list of modules matching
 * @given_alias
 *
 * Create a new list of kmod modules using an alias or module name and lookup
 * libkmod's configuration files and indexes in order to find the module.
 * Once it's found in one of the places, it stops searching and create the
 * list of modules that is saved in @list.
 *
 * The search order is: 1. aliases in configuration file; 2. module names in
 * modules.dep index; 3. symbol aliases in modules.symbols index; 4. aliases
 * in modules.alias index.
 *
 * The initial refcount is 1, and needs to be decremented to release the
 * resources of the kmod_module. The returned @list must be released by
 * calling kmod_module_unref_list(). Since libkmod keeps track of all
 * kmod_modules created, they are all released upon @ctx destruction too. Do
 * not unref @ctx before all the desired operations with the returned list are
 * completed.
 *
 * Returns: 0 on success or < 0 otherwise. It fails if any of the lookup
 * methods failed, which is basically due to memory allocation fail. If module
 * is not found, it still returns 0, but @list is an empty list.
 */
KMOD_EXPORT int kmod_module_new_from_lookup(struct kmod_ctx *ctx,
						const char *given_alias,
						struct kmod_list **list)
{
	int err;
	char alias[PATH_MAX];

	if (ctx == NULL || given_alias == NULL)
		return -ENOENT;

	if (list == NULL || *list != NULL) {
		ERR(ctx, "An empty list is needed to create lookup\n");
		return -ENOSYS;
	}

	if (alias_normalize(given_alias, alias, NULL) < 0) {
		DBG(ctx, "invalid alias: %s\n", given_alias);
		return -EINVAL;
	}

	DBG(ctx, "input alias=%s, normalized=%s\n", given_alias, alias);

	/* Aliases from config file override all the others */
	err = kmod_lookup_alias_from_config(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup modules.dep %s\n", alias);
	err = kmod_lookup_alias_from_moddep_file(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup modules.symbols %s\n", alias);
	err = kmod_lookup_alias_from_symbols_file(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup install and remove commands %s\n", alias);
	err = kmod_lookup_alias_from_commands(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup modules.aliases %s\n", alias);
	err = kmod_lookup_alias_from_aliases_file(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup modules.builtin %s\n", alias);
	err = kmod_lookup_alias_from_builtin_file(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

finish:
	DBG(ctx, "lookup %s=%d, list=%p\n", alias, err, *list);
	return err;
fail:
	DBG(ctx, "Failed to lookup %s\n", alias);
	kmod_module_unref_list(*list);
	*list = NULL;
	return err;
}
#undef CHECK_ERR_AND_FINISH

/**
 * kmod_module_unref_list:
 * @list: list of kmod modules
 *
 * Drop a reference of each kmod module in @list and releases the resources
 * taken by the list itself.
 *
 * Returns: 0
 */
KMOD_EXPORT int kmod_module_unref_list(struct kmod_list *list)
{
	for (; list != NULL; list = kmod_list_remove(list))
		kmod_module_unref(list->data);

	return 0;
}

/**
 * kmod_module_get_filtered_blacklist:
 * @ctx: kmod library context
 * @input: list of kmod_module to be filtered with blacklist
 * @output: where to save the new list
 *
 * This function should not be used. Use kmod_module_apply_filter instead.
 *
 * Given a list @input, this function filter it out with config's blacklist
 * and save it in @output.
 *
 * Returns: 0 on success or < 0 otherwise. @output is saved with the updated
 * list.
 */
KMOD_EXPORT int kmod_module_get_filtered_blacklist(const struct kmod_ctx *ctx,
						const struct kmod_list *input,
						struct kmod_list **output)
{
	return kmod_module_apply_filter(ctx, KMOD_FILTER_BLACKLIST, input, output);
}

static const struct kmod_list *module_get_dependencies_noref(const struct kmod_module *mod)
{
	if (!mod->init.dep) {
		/* lazy init */
		char *line = kmod_search_moddep(mod->ctx, mod->name);

		if (line == NULL)
			return NULL;

		kmod_module_parse_depline((struct kmod_module *)mod, line);
		free(line);

		if (!mod->init.dep)
			return NULL;
	}

	return mod->dep;
}

/**
 * kmod_module_get_dependencies:
 * @mod: kmod module
 *
 * Search the modules.dep index to find the dependencies of the given @mod.
 * The result is cached in @mod, so subsequent calls to this function will
 * return the already searched list of modules.
 *
 * Returns: NULL on failure. Otherwise it returns a list of kmod modules
 * that can be released by calling kmod_module_unref_list().
 */
KMOD_EXPORT struct kmod_list *kmod_module_get_dependencies(const struct kmod_module *mod)
{
	struct kmod_list *l, *l_new, *list_new = NULL;

	if (mod == NULL)
		return NULL;

	module_get_dependencies_noref(mod);

	kmod_list_foreach(l, mod->dep) {
		l_new = kmod_list_append(list_new, kmod_module_ref(l->data));
		if (l_new == NULL) {
			kmod_module_unref(l->data);
			goto fail;
		}

		list_new = l_new;
	}

	return list_new;

fail:
	ERR(mod->ctx, "out of memory\n");
	kmod_module_unref_list(list_new);
	return NULL;
}

/**
 * kmod_module_get_module:
 * @entry: an entry in a list of kmod modules.
 *
 * Get the kmod module of this @entry in the list, increasing its refcount.
 * After it's used, unref it. Since the refcount is incremented upon return,
 * you still have to call kmod_module_unref_list() to release the list of kmod
 * modules.
 *
 * Returns: NULL on failure or the kmod_module contained in this list entry
 * with its refcount incremented.
 */
KMOD_EXPORT struct kmod_module *kmod_module_get_module(const struct kmod_list *entry)
{
	if (entry == NULL)
		return NULL;

	return kmod_module_ref(entry->data);
}

/**
 * kmod_module_get_name:
 * @mod: kmod module
 *
 * Get the name of this kmod module. Name is always available, independently
 * if it was created by kmod_module_new_from_name() or another function and
 * it's always normalized (dashes are replaced with underscores).
 *
 * Returns: the name of this kmod module.
 */
KMOD_EXPORT const char *kmod_module_get_name(const struct kmod_module *mod)
{
	if (mod == NULL)
		return NULL;

	return mod->name;
}

/**
 * kmod_module_get_path:
 * @mod: kmod module
 *
 * Get the path of this kmod module. If this kmod module was not created by
 * path, it can search the modules.dep index in order to find out the module
 * under context's dirname.
 *
 * Returns: the path of this kmod module or NULL if such information is not
 * available.
 */
KMOD_EXPORT const char *kmod_module_get_path(const struct kmod_module *mod)
{
	char *line;

	if (mod == NULL)
		return NULL;

	DBG(mod->ctx, "name='%s' path='%s'\n", mod->name, mod->path);

	if (mod->path != NULL)
		return mod->path;
	if (mod->init.dep)
		return NULL;

	/* lazy init */
	line = kmod_search_moddep(mod->ctx, mod->name);
	if (line == NULL)
		return NULL;

	kmod_module_parse_depline((struct kmod_module *) mod, line);
	free(line);

	return mod->path;
}


extern long delete_module(const char *name, unsigned int flags);

/**
 * kmod_module_remove_module:
 * @mod: kmod module
 * @flags: flags to pass to Linux kernel when removing the module. The only valid flag is
 * KMOD_REMOVE_FORCE: force remove module regardless if it's still in
 * use by a kernel subsystem or other process;
 * KMOD_REMOVE_NOWAIT is always enforced, causing us to pass O_NONBLOCK to
 * delete_module(2).
 *
 * Remove a module from Linux kernel.
 *
 * Returns: 0 on success or < 0 on failure.
 */
KMOD_EXPORT int kmod_module_remove_module(struct kmod_module *mod,
							unsigned int flags)
{
	int err;

	if (mod == NULL)
		return -ENOENT;

	/* Filter out other flags and force ONONBLOCK */
	flags &= KMOD_REMOVE_FORCE;
	flags |= KMOD_REMOVE_NOWAIT;

	err = delete_module(mod->name, flags);
	if (err != 0) {
		err = -errno;
		ERR(mod->ctx, "could not remove '%s': %m\n", mod->name);
	}

	return err;
}

extern long init_module(const void *mem, unsigned long len, const char *args);

/**
 * kmod_module_insert_module:
 * @mod: kmod module
 * @flags: flags are not passed to Linux Kernel, but instead they dictate the
 * behavior of this function, valid flags are
 * KMOD_INSERT_FORCE_VERMAGIC: ignore kernel version magic;
 * KMOD_INSERT_FORCE_MODVERSION: ignore symbol version hashes.
 * @options: module's options to pass to Linux Kernel.
 *
 * Insert a module in Linux kernel. It opens the file pointed by @mod,
 * mmap'ing it and passing to kernel.
 *
 * Returns: 0 on success or < 0 on failure. If module is already loaded it
 * returns -EEXIST.
 */
KMOD_EXPORT int kmod_module_insert_module(struct kmod_module *mod,
							unsigned int flags,
							const char *options)
{
	int err;
	const void *mem;
	off_t size;
	struct kmod_elf *elf;
	const char *path;
	const char *args = options ? options : "";

	if (mod == NULL)
		return -ENOENT;

	path = kmod_module_get_path(mod);
	if (path == NULL) {
		ERR(mod->ctx, "could not find module by name='%s'\n", mod->name);
		return -ENOENT;
	}

	mod->file = kmod_file_open(mod->ctx, path);
	if (mod->file == NULL) {
		err = -errno;
		return err;
	}

	if (kmod_file_get_direct(mod->file)) {
		unsigned int kernel_flags = 0;

		if (flags & KMOD_INSERT_FORCE_VERMAGIC)
			kernel_flags |= MODULE_INIT_IGNORE_VERMAGIC;
		if (flags & KMOD_INSERT_FORCE_MODVERSION)
			kernel_flags |= MODULE_INIT_IGNORE_MODVERSIONS;

		err = finit_module(kmod_file_get_fd(mod->file), args, kernel_flags);
		if (err == 0 || errno != ENOSYS)
			goto init_finished;
	}

	if (flags & (KMOD_INSERT_FORCE_VERMAGIC | KMOD_INSERT_FORCE_MODVERSION)) {<commit_msg>module: fix a memory leak<commit_after>/*
 * libkmod - interface to kernel module operations
 *
 * Copyright (C) 2011-2013  ProFUSION embedded systems
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

#include <assert.h>
#include <ctype.h>
#include <dirent.h>
#include <errno.h>
#include <fnmatch.h>
#include <inttypes.h>
#include <limits.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <sys/wait.h>
#ifdef HAVE_LINUX_MODULE_H
#include <linux/module.h>
#endif

#include <shared/util.h>

#include "libkmod.h"
#include "libkmod-internal.h"

/**
 * SECTION:libkmod-module
 * @short_description: operate on kernel modules
 */

enum kmod_module_builtin {
    KMOD_MODULE_BUILTIN_UNKNOWN,
    KMOD_MODULE_BUILTIN_NO,
    KMOD_MODULE_BUILTIN_YES,
};

/**
 * kmod_module:
 *
 * Opaque object representing a module.
 */
struct kmod_module {
	struct kmod_ctx *ctx;
	char *hashkey;
	char *name;
	char *path;
	struct kmod_list *dep;
	char *options;
	const char *install_commands;	/* owned by kmod_config */
	const char *remove_commands;	/* owned by kmod_config */
	char *alias; /* only set if this module was created from an alias */
	struct kmod_file *file;
	int n_dep;
	int refcount;
	struct {
		bool dep : 1;
		bool options : 1;
		bool install_commands : 1;
		bool remove_commands : 1;
	} init;

	/*
	 * mark if module is builtin, i.e. it's present on modules.builtin
	 * file. This is set as soon as it is needed or as soon as we know
	 * about it, i.e. the module was created from builtin lookup.
	 */
	enum kmod_module_builtin builtin;

	/*
	 * private field used by kmod_module_get_probe_list() to detect
	 * dependency loops
	 */
	bool visited : 1;

	/*
	 * set by kmod_module_get_probe_list: indicates for probe_insert()
	 * whether the module's command and softdep should be ignored
	 */
	bool ignorecmd : 1;

	/*
	 * set by kmod_module_get_probe_list: indicates whether this is the
	 * module the user asked for or its dependency, or whether this
	 * is a softdep only
	 */
	bool required : 1;
};

static inline const char *path_join(const char *path, size_t prefixlen,
							char buf[PATH_MAX])
{
	size_t pathlen;

	if (path[0] == '/')
		return path;

	pathlen = strlen(path);
	if (prefixlen + pathlen + 1 >= PATH_MAX)
		return NULL;

	memcpy(buf + prefixlen, path, pathlen + 1);
	return buf;
}

static inline bool module_is_inkernel(struct kmod_module *mod)
{
	int state = kmod_module_get_initstate(mod);

	if (state == KMOD_MODULE_LIVE ||
			state == KMOD_MODULE_BUILTIN)
		return true;

	return false;
}

int kmod_module_parse_depline(struct kmod_module *mod, char *line)
{
	struct kmod_ctx *ctx = mod->ctx;
	struct kmod_list *list = NULL;
	const char *dirname;
	char buf[PATH_MAX];
	char *p, *saveptr;
	int err = 0, n = 0;
	size_t dirnamelen;

	if (mod->init.dep)
		return mod->n_dep;
	assert(mod->dep == NULL);
	mod->init.dep = true;

	p = strchr(line, ':');
	if (p == NULL)
		return 0;

	*p = '\0';
	dirname = kmod_get_dirname(mod->ctx);
	dirnamelen = strlen(dirname);
	if (dirnamelen + 2 >= PATH_MAX)
		return 0;

	memcpy(buf, dirname, dirnamelen);
	buf[dirnamelen] = '/';
	dirnamelen++;
	buf[dirnamelen] = '\0';

	if (mod->path == NULL) {
		const char *str = path_join(line, dirnamelen, buf);
		if (str == NULL)
			return 0;
		mod->path = strdup(str);
		if (mod->path == NULL)
			return 0;
	}

	p++;
	for (p = strtok_r(p, " \t", &saveptr); p != NULL;
					p = strtok_r(NULL, " \t", &saveptr)) {
		struct kmod_module *depmod = NULL;
		const char *path;

		path = path_join(p, dirnamelen, buf);
		if (path == NULL) {
			ERR(ctx, "could not join path '%s' and '%s'.\n",
			    dirname, p);
			goto fail;
		}

		err = kmod_module_new_from_path(ctx, path, &depmod);
		if (err < 0) {
			ERR(ctx, "ctx=%p path=%s error=%s\n",
						ctx, path, strerror(-err));
			goto fail;
		}

		DBG(ctx, "add dep: %s\n", path);

		list = kmod_list_prepend(list, depmod);
		n++;
	}

	DBG(ctx, "%d dependencies for %s\n", n, mod->name);

	mod->dep = list;
	mod->n_dep = n;
	return n;

fail:
	kmod_module_unref_list(list);
	mod->init.dep = false;
	return err;
}

void kmod_module_set_visited(struct kmod_module *mod, bool visited)
{
	mod->visited = visited;
}

void kmod_module_set_builtin(struct kmod_module *mod, bool builtin)
{
	mod->builtin =
		builtin ? KMOD_MODULE_BUILTIN_YES : KMOD_MODULE_BUILTIN_NO;
}

void kmod_module_set_required(struct kmod_module *mod, bool required)
{
	mod->required = required;
}

bool kmod_module_is_builtin(struct kmod_module *mod)
{
	if (mod->builtin == KMOD_MODULE_BUILTIN_UNKNOWN) {
		kmod_module_set_builtin(mod,
					kmod_lookup_alias_is_builtin(mod->ctx, mod->name));
	}

	return mod->builtin == KMOD_MODULE_BUILTIN_YES;
}
/*
 * Memory layout with alias:
 *
 * struct kmod_module {
 *        hashkey -----.
 *        alias -----. |
 *        name ----. | |
 * }               | | |
 * name <----------' | |
 * alias <-----------' |
 * name\alias <--------'
 *
 * Memory layout without alias:
 *
 * struct kmod_module {
 *        hashkey ---.
 *        alias -----|----> NULL
 *        name ----. |
 * }               | |
 * name <----------'-'
 *
 * @key is "name\alias" or "name" (in which case alias == NULL)
 */
static int kmod_module_new(struct kmod_ctx *ctx, const char *key,
				const char *name, size_t namelen,
				const char *alias, size_t aliaslen,
				struct kmod_module **mod)
{
	struct kmod_module *m;
	size_t keylen;

	m = kmod_pool_get_module(ctx, key);
	if (m != NULL) {
		*mod = kmod_module_ref(m);
		return 0;
	}

	if (alias == NULL)
		keylen = namelen;
	else
		keylen = namelen + aliaslen + 1;

	m = malloc(sizeof(*m) + (alias == NULL ? 1 : 2) * (keylen + 1));
	if (m == NULL)
		return -ENOMEM;

	memset(m, 0, sizeof(*m));

	m->ctx = kmod_ref(ctx);
	m->name = (char *)m + sizeof(*m);
	memcpy(m->name, key, keylen + 1);
	if (alias == NULL) {
		m->hashkey = m->name;
		m->alias = NULL;
	} else {
		m->name[namelen] = '\0';
		m->alias = m->name + namelen + 1;
		m->hashkey = m->name + keylen + 1;
		memcpy(m->hashkey, key, keylen + 1);
	}

	m->refcount = 1;
	kmod_pool_add_module(ctx, m, m->hashkey);
	*mod = m;

	return 0;
}

/**
 * kmod_module_new_from_name:
 * @ctx: kmod library context
 * @name: name of the module
 * @mod: where to save the created struct kmod_module
 *
 * Create a new struct kmod_module using the module name. @name can not be an
 * alias, file name or anything else; it must be a module name. There's no
 * check if the module exists in the system.
 *
 * This function is also used internally by many others that return a new
 * struct kmod_module or a new list of modules.
 *
 * The initial refcount is 1, and needs to be decremented to release the
 * resources of the kmod_module. Since libkmod keeps track of all
 * kmod_modules created, they are all released upon @ctx destruction too. Do
 * not unref @ctx before all the desired operations with the returned
 * kmod_module are done.
 *
 * Returns: 0 on success or < 0 otherwise. It fails if name is not a valid
 * module name or if memory allocation failed.
 */
KMOD_EXPORT int kmod_module_new_from_name(struct kmod_ctx *ctx,
						const char *name,
						struct kmod_module **mod)
{
	size_t namelen;
	char name_norm[PATH_MAX];

	if (ctx == NULL || name == NULL || mod == NULL)
		return -ENOENT;

	modname_normalize(name, name_norm, &namelen);

	return kmod_module_new(ctx, name_norm, name_norm, namelen, NULL, 0, mod);
}

int kmod_module_new_from_alias(struct kmod_ctx *ctx, const char *alias,
				const char *name, struct kmod_module **mod)
{
	int err;
	char key[PATH_MAX];
	size_t namelen = strlen(name);
	size_t aliaslen = strlen(alias);

	if (namelen + aliaslen + 2 > PATH_MAX)
		return -ENAMETOOLONG;

	memcpy(key, name, namelen);
	memcpy(key + namelen + 1, alias, aliaslen + 1);
	key[namelen] = '\\';

	err = kmod_module_new(ctx, key, name, namelen, alias, aliaslen, mod);
	if (err < 0)
		return err;

	return 0;
}

/**
 * kmod_module_new_from_path:
 * @ctx: kmod library context
 * @path: path where to find the given module
 * @mod: where to save the created struct kmod_module
 *
 * Create a new struct kmod_module using the module path. @path must be an
 * existent file with in the filesystem and must be accessible to libkmod.
 *
 * The initial refcount is 1, and needs to be decremented to release the
 * resources of the kmod_module. Since libkmod keeps track of all
 * kmod_modules created, they are all released upon @ctx destruction too. Do
 * not unref @ctx before all the desired operations with the returned
 * kmod_module are done.
 *
 * If @path is relative, it's treated as relative to the current working
 * directory. Otherwise, give an absolute path.
 *
 * Returns: 0 on success or < 0 otherwise. It fails if file does not exist, if
 * it's not a valid file for a kmod_module or if memory allocation failed.
 */
KMOD_EXPORT int kmod_module_new_from_path(struct kmod_ctx *ctx,
						const char *path,
						struct kmod_module **mod)
{
	struct kmod_module *m;
	int err;
	struct stat st;
	char name[PATH_MAX];
	char *abspath;
	size_t namelen;

	if (ctx == NULL || path == NULL || mod == NULL)
		return -ENOENT;

	abspath = path_make_absolute_cwd(path);
	if (abspath == NULL) {
		DBG(ctx, "no absolute path for %s\n", path);
		return -ENOMEM;
	}

	err = stat(abspath, &st);
	if (err < 0) {
		err = -errno;
		DBG(ctx, "stat %s: %s\n", path, strerror(errno));
		free(abspath);
		return err;
	}

	if (path_to_modname(path, name, &namelen) == NULL) {
		DBG(ctx, "could not get modname from path %s\n", path);
		free(abspath);
		return -ENOENT;
	}

	m = kmod_pool_get_module(ctx, name);
	if (m != NULL) {
		if (m->path == NULL)
			m->path = abspath;
		else if (streq(m->path, abspath))
			free(abspath);
		else {
			ERR(ctx, "kmod_module '%s' already exists with different path: new-path='%s' old-path='%s'\n",
							name, abspath, m->path);
			free(abspath);
			return -EEXIST;
		}

		*mod = kmod_module_ref(m);
		return 0;
	}

	err = kmod_module_new(ctx, name, name, namelen, NULL, 0, &m);
	if (err < 0) {
		free(abspath);
		return err;
	}

	m->path = abspath;
	*mod = m;

	return 0;
}

/**
 * kmod_module_unref:
 * @mod: kmod module
 *
 * Drop a reference of the kmod module. If the refcount reaches zero, its
 * resources are released.
 *
 * Returns: NULL if @mod is NULL or if the module was released. Otherwise it
 * returns the passed @mod with its refcount decremented.
 */
KMOD_EXPORT struct kmod_module *kmod_module_unref(struct kmod_module *mod)
{
	if (mod == NULL)
		return NULL;

	if (--mod->refcount > 0)
		return mod;

	DBG(mod->ctx, "kmod_module %p released\n", mod);

	kmod_pool_del_module(mod->ctx, mod, mod->hashkey);
	kmod_module_unref_list(mod->dep);

	if (mod->file)
		kmod_file_unref(mod->file);

	kmod_unref(mod->ctx);
	free(mod->options);
	free(mod->path);
	free(mod);
	return NULL;
}

/**
 * kmod_module_ref:
 * @mod: kmod module
 *
 * Take a reference of the kmod module, incrementing its refcount.
 *
 * Returns: the passed @module with its refcount incremented.
 */
KMOD_EXPORT struct kmod_module *kmod_module_ref(struct kmod_module *mod)
{
	if (mod == NULL)
		return NULL;

	mod->refcount++;

	return mod;
}

#define CHECK_ERR_AND_FINISH(_err, _label_err, _list, label_finish)	\
	do {								\
		if ((_err) < 0)						\
			goto _label_err;				\
		if (*(_list) != NULL)					\
			goto finish;					\
	} while (0)

/**
 * kmod_module_new_from_lookup:
 * @ctx: kmod library context
 * @given_alias: alias to look for
 * @list: an empty list where to save the list of modules matching
 * @given_alias
 *
 * Create a new list of kmod modules using an alias or module name and lookup
 * libkmod's configuration files and indexes in order to find the module.
 * Once it's found in one of the places, it stops searching and create the
 * list of modules that is saved in @list.
 *
 * The search order is: 1. aliases in configuration file; 2. module names in
 * modules.dep index; 3. symbol aliases in modules.symbols index; 4. aliases
 * in modules.alias index.
 *
 * The initial refcount is 1, and needs to be decremented to release the
 * resources of the kmod_module. The returned @list must be released by
 * calling kmod_module_unref_list(). Since libkmod keeps track of all
 * kmod_modules created, they are all released upon @ctx destruction too. Do
 * not unref @ctx before all the desired operations with the returned list are
 * completed.
 *
 * Returns: 0 on success or < 0 otherwise. It fails if any of the lookup
 * methods failed, which is basically due to memory allocation fail. If module
 * is not found, it still returns 0, but @list is an empty list.
 */
KMOD_EXPORT int kmod_module_new_from_lookup(struct kmod_ctx *ctx,
						const char *given_alias,
						struct kmod_list **list)
{
	int err;
	char alias[PATH_MAX];

	if (ctx == NULL || given_alias == NULL)
		return -ENOENT;

	if (list == NULL || *list != NULL) {
		ERR(ctx, "An empty list is needed to create lookup\n");
		return -ENOSYS;
	}

	if (alias_normalize(given_alias, alias, NULL) < 0) {
		DBG(ctx, "invalid alias: %s\n", given_alias);
		return -EINVAL;
	}

	DBG(ctx, "input alias=%s, normalized=%s\n", given_alias, alias);

	/* Aliases from config file override all the others */
	err = kmod_lookup_alias_from_config(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup modules.dep %s\n", alias);
	err = kmod_lookup_alias_from_moddep_file(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup modules.symbols %s\n", alias);
	err = kmod_lookup_alias_from_symbols_file(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup install and remove commands %s\n", alias);
	err = kmod_lookup_alias_from_commands(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup modules.aliases %s\n", alias);
	err = kmod_lookup_alias_from_aliases_file(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

	DBG(ctx, "lookup modules.builtin %s\n", alias);
	err = kmod_lookup_alias_from_builtin_file(ctx, alias, list);
	CHECK_ERR_AND_FINISH(err, fail, list, finish);

finish:
	DBG(ctx, "lookup %s=%d, list=%p\n", alias, err, *list);
	return err;
fail:
	DBG(ctx, "Failed to lookup %s\n", alias);
	kmod_module_unref_list(*list);
	*list = NULL;
	return err;
}
#undef CHECK_ERR_AND_FINISH

/**
 * kmod_module_unref_list:
 * @list: list of kmod modules
 *
 * Drop a reference of each kmod module in @list and releases the resources
 * taken by the list itself.
 *
 * Returns: 0
 */
KMOD_EXPORT int kmod_module_unref_list(struct kmod_list *list)
{
	for (; list != NULL; list = kmod_list_remove(list))
		kmod_module_unref(list->data);

	return 0;
}

/**
 * kmod_module_get_filtered_blacklist:
 * @ctx: kmod library context
 * @input: list of kmod_module to be filtered with blacklist
 * @output: where to save the new list
 *
 * This function should not be used. Use kmod_module_apply_filter instead.
 *
 * Given a list @input, this function filter it out with config's blacklist
 * and save it in @output.
 *
 * Returns: 0 on success or < 0 otherwise. @output is saved with the updated
 * list.
 */
KMOD_EXPORT int kmod_module_get_filtered_blacklist(const struct kmod_ctx *ctx,
						const struct kmod_list *input,
						struct kmod_list **output)
{
	return kmod_module_apply_filter(ctx, KMOD_FILTER_BLACKLIST, input, output);
}

static const struct kmod_list *module_get_dependencies_noref(const struct kmod_module *mod)
{
	if (!mod->init.dep) {
		/* lazy init */
		char *line = kmod_search_moddep(mod->ctx, mod->name);

		if (line == NULL)
			return NULL;

		kmod_module_parse_depline((struct kmod_module *)mod, line);
		free(line);

		if (!mod->init.dep)
			return NULL;
	}

	return mod->dep;
}

/**
 * kmod_module_get_dependencies:
 * @mod: kmod module
 *
 * Search the modules.dep index to find the dependencies of the given @mod.
 * The result is cached in @mod, so subsequent calls to this function will
 * return the already searched list of modules.
 *
 * Returns: NULL on failure. Otherwise it returns a list of kmod modules
 * that can be released by calling kmod_module_unref_list().
 */
KMOD_EXPORT struct kmod_list *kmod_module_get_dependencies(const struct kmod_module *mod)
{
	struct kmod_list *l, *l_new, *list_new = NULL;

	if (mod == NULL)
		return NULL;

	module_get_dependencies_noref(mod);

	kmod_list_foreach(l, mod->dep) {
		l_new = kmod_list_append(list_new, kmod_module_ref(l->data));
		if (l_new == NULL) {
			kmod_module_unref(l->data);
			goto fail;
		}

		list_new = l_new;
	}

	return list_new;

fail:
	ERR(mod->ctx, "out of memory\n");
	kmod_module_unref_list(list_new);
	return NULL;
}

/**
 * kmod_module_get_module:
 * @entry: an entry in a list of kmod modules.
 *
 * Get the kmod module of this @entry in the list, increasing its refcount.
 * After it's used, unref it. Since the refcount is incremented upon return,
 * you still have to call kmod_module_unref_list() to release the list of kmod
 * modules.
 *
 * Returns: NULL on failure or the kmod_module contained in this list entry
 * with its refcount incremented.
 */
KMOD_EXPORT struct kmod_module *kmod_module_get_module(const struct kmod_list *entry)
{
	if (entry == NULL)
		return NULL;

	return kmod_module_ref(entry->data);
}

/**
 * kmod_module_get_name:
 * @mod: kmod module
 *
 * Get the name of this kmod module. Name is always available, independently
 * if it was created by kmod_module_new_from_name() or another function and
 * it's always normalized (dashes are replaced with underscores).
 *
 * Returns: the name of this kmod module.
 */
KMOD_EXPORT const char *kmod_module_get_name(const struct kmod_module *mod)
{
	if (mod == NULL)
		return NULL;

	return mod->name;
}

/**
 * kmod_module_get_path:
 * @mod: kmod module
 *
 * Get the path of this kmod module. If this kmod module was not created by
 * path, it can search the modules.dep index in order to find out the module
 * under context's dirname.
 *
 * Returns: the path of this kmod module or NULL if such information is not
 * available.
 */
KMOD_EXPORT const char *kmod_module_get_path(const struct kmod_module *mod)
{
	char *line;

	if (mod == NULL)
		return NULL;

	DBG(mod->ctx, "name='%s' path='%s'\n", mod->name, mod->path);

	if (mod->path != NULL)
		return mod->path;
	if (mod->init.dep)
		return NULL;

	/* lazy init */
	line = kmod_search_moddep(mod->ctx, mod->name);
	if (line == NULL)
		return NULL;

	kmod_module_parse_depline((struct kmod_module *) mod, line);
	free(line);

	return mod->path;
}


extern long delete_module(const char *name, unsigned int flags);

/**
 * kmod_module_remove_module:
 * @mod: kmod module
 * @flags: flags to pass to Linux kernel when removing the module. The only valid flag is
 * KMOD_REMOVE_FORCE: force remove module regardless if it's still in
 * use by a kernel subsystem or other process;
 * KMOD_REMOVE_NOWAIT is always enforced, causing us to pass O_NONBLOCK to
 * delete_module(2).
 *
 * Remove a module from Linux kernel.
 *
 * Returns: 0 on success or < 0 on failure.
 */
KMOD_EXPORT int kmod_module_remove_module(struct kmod_module *mod,
							unsigned int flags)
{
	int err;

	if (mod == NULL)
		return -ENOENT;

	/* Filter out other flags and force ONONBLOCK */
	flags &= KMOD_REMOVE_FORCE;
	flags |= KMOD_REMOVE_NOWAIT;

	err = delete_module(mod->name, flags);
	if (err != 0) {
		err = -errno;
		ERR(mod->ctx, "could not remove '%s': %m\n", mod->name);
	}

	return err;
}

extern long init_module(const void *mem, unsigned long len, const char *args);

/**
 * kmod_module_insert_module:
 * @mod: kmod module
 * @flags: flags are not passed to Linux Kernel, but instead they dictate the
 * behavior of this function, valid flags are
 * KMOD_INSERT_FORCE_VERMAGIC: ignore kernel version magic;
 * KMOD_INSERT_FORCE_MODVERSION: ignore symbol version hashes.
 * @options: module's options to pass to Linux Kernel.
 *
 * Insert a module in Linux kernel. It opens the file pointed by @mod,
 * mmap'ing it and passing to kernel.
 *
 * Returns: 0 on success or < 0 on failure. If module is already loaded it
 * returns -EEXIST.
 */
KMOD_EXPORT int kmod_module_insert_module(struct kmod_module *mod,
							unsigned int flags,
							const char *options)
{
	int err;
	const void *mem;
	off_t size;
	struct kmod_elf *elf;
	const char *path;
	const char *args = options ? options : "";

	if (mod == NULL)
		return -ENOENT;

	path = kmod_module_get_path(mod);
	if (path == NULL) {
		ERR(mod->ctx, "could not find module by name='%s'\n", mod->name);
		return -ENOENT;
	}

	if (!mod->file) {
		mod->file = kmod_file_open(mod->ctx, path);
		if (mod->file == NULL) {
			err = -errno;
			return err;
		}
	}

	if (kmod_file_get_direct(mod->file)) {
		unsigned int kernel_flags = 0;

		if (flags & KMOD_INSERT_FORCE_VERMAGIC)
			kernel_flags |= MODULE_INIT_IGNORE_VERMAGIC;
		if (flags & KMOD_INSERT_FORCE_MODVERSION)
			kernel_flags |= MODULE_INIT_IGNORE_MODVERSIONS;

		err = finit_module(kmod_file_get_fd(mod->file), args, kernel_flags);
		if (err == 0 || errno != ENOSYS)
			goto init_finished;
	}

	if (flags & (KMOD_INSERT_FORCE_VERMAGIC | KMOD_INSERT_FORCE_MODVERSION)) {
{'fraction_non_alphanumeric': 0.11891619523144224, 'fraction_numerical': 0.003847827580678409, 'mean_word_length': 3.7360885128538888, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 218, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4406275', 'n_tokens_mistral': 16315, 'n_tokens_neox': 15331, 'n_words': 4996}