<commit_before>#include "common.h"
#include "fileops.h"
#include <ctype.h>

int gitfo_mkdir_2file(const char *file_path)
{
	const int mode = 0755; /* or 0777 ? */
	int error = GIT_SUCCESS;
	char target_folder_path[GIT_PATH_MAX];

	error = git__dirname_r(target_folder_path, sizeof(target_folder_path), file_path);
	if (error < GIT_SUCCESS)
		return git__throw(GIT_EINVALIDPATH, "Failed to recursively build `%s` tree structure. Unable to parse parent folder name", file_path);

	/* Does the containing folder exist? */
	if (gitfo_isdir(target_folder_path)) {
		git__joinpath(target_folder_path, target_folder_path, ""); /* Ensure there's a trailing slash */

		/* Let's create the tree structure */
		error = gitfo_mkdir_recurs(target_folder_path, mode);
		if (error < GIT_SUCCESS)
			return error;	/* The callee already takes care of setting the correct error message. */
	}

	return GIT_SUCCESS;
}

int gitfo_mktemp(char *path_out, const char *filename)
{
	int fd;

	strcpy(path_out, filename);
	strcat(path_out, "_git2_XXXXXX");

#if defined(_MSC_VER)
	/* FIXME: there may be race conditions when multi-threading
	 * with the library */
	if (_mktemp_s(path_out, GIT_PATH_MAX) != 0)
		return git__throw(GIT_EOSERR, "Failed to make temporary file %s", path_out);

	fd = gitfo_creat(path_out, 0744);
#else
	fd = mkstemp(path_out);
#endif

	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to create temporary file %s", path_out);
}

int gitfo_open(const char *path, int flags)
{
	int fd = open(path, flags | O_BINARY);
	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to open %s", path);
}

int gitfo_creat(const char *path, int mode)
{
	int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, mode);
	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to create file. Could not open %s", path);
}

int gitfo_creat_force(const char *path, int mode)
{
	if (gitfo_mkdir_2file(path) < GIT_SUCCESS)
		return git__throw(GIT_EOSERR, "Failed to create file %s", path);

	return gitfo_creat(path, mode);
}

int gitfo_creat_locked(const char *path, int mode)
{
	int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_EXCL, mode);
	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to create locked file. Could not open %s", path);
}

int gitfo_creat_locked_force(const char *path, int mode)
{
	if (gitfo_mkdir_2file(path) < GIT_SUCCESS)
		return git__throw(GIT_EOSERR, "Failed to create locked file %s", path);

	return gitfo_creat_locked(path, mode);
}

int gitfo_read(git_file fd, void *buf, size_t cnt)
{
	char *b = buf;
	while (cnt) {
		ssize_t r = read(fd, b, cnt);
		if (r < 0) {
			if (errno == EINTR || errno == EAGAIN)
				continue;
			return git__throw(GIT_EOSERR, "Failed to read from file");
		}
		if (!r) {
			errno = EPIPE;
			return git__throw(GIT_EOSERR, "Failed to read from file");
		}
		cnt -= r;
		b += r;
	}
	return GIT_SUCCESS;
}

int gitfo_write(git_file fd, void *buf, size_t cnt)
{
	char *b = buf;
	while (cnt) {
		ssize_t r = write(fd, b, cnt);
		if (r < 0) {
			if (errno == EINTR || errno == EAGAIN)
				continue;
			return git__throw(GIT_EOSERR, "Failed to write to file");
		}
		if (!r) {
			errno = EPIPE;
			return git__throw(GIT_EOSERR, "Failed to write to file");
		}
		cnt -= r;
		b += r;
	}
	return GIT_SUCCESS;
}

int gitfo_isdir(const char *path)
{
	struct stat st;
	int len, stat_error;

	if (!path)
		return git__throw(GIT_ENOTFOUND, "No path given to gitfo_isdir");

	len = strlen(path);

	/* win32: stat path for folders cannot end in a slash */
	if (path[len - 1] == '/') {
		char *path_fixed = NULL;
		path_fixed = git__strdup(path);
		path_fixed[len - 1] = 0;
		stat_error = gitfo_stat(path_fixed, &st);
		free(path_fixed);
	} else {
		stat_error = gitfo_stat(path, &st);
	}

	if (stat_error < GIT_SUCCESS)
		return git__throw(GIT_ENOTFOUND, "%s does not exist", path);

	if (!S_ISDIR(st.st_mode))
		return git__throw(GIT_ENOTFOUND, "%s is not a directory", path);

	return GIT_SUCCESS;
}

int gitfo_isfile(const char *path)
{
	struct stat st;
	int stat_error;

	if (!path)
		return git__throw(GIT_ENOTFOUND, "No path given to gitfo_isfile");

	stat_error = gitfo_stat(path, &st);

	if (stat_error < GIT_SUCCESS)
		return git__throw(GIT_ENOTFOUND, "%s does not exist", path);

	if (!S_ISREG(st.st_mode))
		return git__throw(GIT_ENOTFOUND, "%s is not a file", path);

	return GIT_SUCCESS;
}

int gitfo_exists(const char *path)
{
	assert(path);
	return access(path, F_OK);
}

git_off_t gitfo_size(git_file fd)
{
	struct stat sb;
	if (gitfo_fstat(fd, &sb))
		return git__throw(GIT_EOSERR, "Failed to get size of file. File missing or corrupted");
	return sb.st_size;
}

int gitfo_read_file(gitfo_buf *obj, const char *path)
{
	git_file fd;
	size_t len;
	git_off_t size;
	unsigned char *buff;

	assert(obj && path && *path);

	if ((fd = gitfo_open(path, O_RDONLY)) < 0)
		return git__throw(GIT_ERROR, "Failed to open %s for reading", path);

	if (((size = gitfo_size(fd)) < 0) || !git__is_sizet(size+1)) {
		gitfo_close(fd);
		return git__throw(GIT_ERROR, "Failed to read file `%s`. An error occured while calculating its size", path);
	}
	len = (size_t) size;

	if ((buff = git__malloc(len + 1)) == NULL) {
		gitfo_close(fd);
		return GIT_ENOMEM;
	}

	if (gitfo_read(fd, buff, len) < 0) {
		gitfo_close(fd);
		free(buff);
		return git__throw(GIT_ERROR, "Failed to read file `%s`", path);
	}
	buff[len] = '\0';

	gitfo_close(fd);

	obj->data = buff;
	obj->len  = len;

	return GIT_SUCCESS;
}

void gitfo_free_buf(gitfo_buf *obj)
{
	assert(obj);
	free(obj->data);
	obj->data = NULL;
}

int gitfo_mv(const char *from, const char *to)
{
	int error;

#ifdef GIT_WIN32
	/*
	 * Win32 POSIX compilance my ass. If the destination
	 * file exists, the `rename` call fails. This is as
	 * close as it gets with the Win32 API.
	 */
	error = MoveFileEx(from, to, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) ? GIT_SUCCESS : GIT_EOSERR;
#else
	/* Don't even try this on Win32 */
	if (!link(from, to)) {
		gitfo_unlink(from);
		return GIT_SUCCESS;
	}

	if (!rename(from, to))
		return GIT_SUCCESS;

	error = GIT_EOSERR;
#endif

	if (error < GIT_SUCCESS)
		return git__throw(error, "Failed to move file from `%s` to `%s`", from, to);

	return GIT_SUCCESS;
}

int gitfo_mv_force(const char *from, const char *to)
{
	if (gitfo_mkdir_2file(to) < GIT_SUCCESS)
		return GIT_EOSERR;	/* The callee already takes care of setting the correct error message. */

	return gitfo_mv(from, to);	/* The callee already takes care of setting the correct error message. */
}

int gitfo_map_ro(git_map *out, git_file fd, git_off_t begin, size_t len)
{
	if (git__mmap(out, len, GIT_PROT_READ, GIT_MAP_SHARED, fd, begin) < GIT_SUCCESS)
		return GIT_EOSERR;
	return GIT_SUCCESS;
}

void gitfo_free_map(git_map *out)
{
	git__munmap(out);
}

int gitfo_dirent(
	char *path,
	size_t path_sz,
	int (*fn)(void *, char *),
	void *arg)
{
	size_t wd_len = strlen(path);
	DIR *dir;
	struct dirent *de;

	if (!wd_len || path_sz < wd_len + 2)
		return git__throw(GIT_EINVALIDARGS, "Failed to process `%s` tree structure. Path is either empty or buffer size is too short", path);

	while (path[wd_len - 1] == '/')
		wd_len--;
	path[wd_len++] = '/';
	path[wd_len] = '\0';

	dir = opendir(path);
	if (!dir)
		return git__throw(GIT_EOSERR, "Failed to process `%s` tree structure. An error occured while opening the directory", path);

	while ((de = readdir(dir)) != NULL) {
		size_t de_len;
		int result;

		/* always skip '.' and '..' */
		if (de->d_name[0] == '.') {
			if (de->d_name[1] == '\0')
				continue;
			if (de->d_name[1] == '.' && de->d_name[2] == '\0')
				continue;
		}

		de_len = strlen(de->d_name);
		if (path_sz < wd_len + de_len + 1) {
			closedir(dir);
			return git__throw(GIT_ERROR, "Failed to process `%s` tree structure. Buffer size is too short", path);
		}

		strcpy(path + wd_len, de->d_name);
		result = fn(arg, path);
		if (result < GIT_SUCCESS) {
			closedir(dir);
			return result;	/* The callee is reponsible for setting the correct error message */
		}
		if (result > 0) {
			closedir(dir);
			return result;
		}
	}

	closedir(dir);
	return GIT_SUCCESS;
}

void gitfo_posixify_path(char *path)
{
	#if GIT_PLATFORM_PATH_SEP != '/'
		while (*path) {
			if (*path == GIT_PLATFORM_PATH_SEP)
				*path = '/';

			path++;
		}
	#endif
}

int gitfo_retrieve_path_root_offset(const char *path)
{
	int offset = 0;

#ifdef GIT_WIN32

	/* Does the root of the path look like a windows drive ? */
	if (isalpha(path[0]) && (path[1] == ':'))
		offset += 2;

#endif

	if (*(path + offset) == '/')
		return offset;

	return -1;	/* Not a real error. Rather a signal than the path is not rooted */
}

int gitfo_mkdir_recurs(const char *path, int mode)
{
	int error, root_path_offset;
	char *pp, *sp;
    char *path_copy = git__strdup(path);

	if (path_copy == NULL)
		return GIT_ENOMEM;

	error = GIT_SUCCESS;
	pp = path_copy;

	root_path_offset = gitfo_retrieve_path_root_offset(pp);
	if (root_path_offset > 0)
		pp += root_path_offset; /* On Windows, will skip the drive name (eg. C: or D:) */

    while (error == GIT_SUCCESS && (sp = strchr(pp, '/')) != 0) {
		if (sp != pp && gitfo_isdir(path_copy) < GIT_SUCCESS) {
			*sp = 0;
			error = gitfo_mkdir(path_copy, mode);

			/* Do not choke while trying to recreate an existing directory */
			if (errno == EEXIST)
				error = GIT_SUCCESS;

			*sp = '/';
		}

		pp = sp + 1;
	}

	if (*(pp - 1) != '/' && error == GIT_SUCCESS)
		error = gitfo_mkdir(path, mode);

	free(path_copy);

	if (error < GIT_SUCCESS)
		return git__throw(error, "Failed to recursively create `%s` tree structure", path);

	return GIT_SUCCESS;
}

static int retrieve_previous_path_component_start(const char *path)
{
	int offset, len, root_offset, start = 0;

	root_offset = gitfo_retrieve_path_root_offset(path);
	if (root_offset > -1)<commit_msg>Fileops: Fixed gitfo_mkdir_recurs so that it proprely works with a path without trailing slash.<commit_after>#include "common.h"
#include "fileops.h"
#include <ctype.h>

int gitfo_mkdir_2file(const char *file_path)
{
	const int mode = 0755; /* or 0777 ? */
	int error = GIT_SUCCESS;
	char target_folder_path[GIT_PATH_MAX];

	error = git__dirname_r(target_folder_path, sizeof(target_folder_path), file_path);
	if (error < GIT_SUCCESS)
		return git__throw(GIT_EINVALIDPATH, "Failed to recursively build `%s` tree structure. Unable to parse parent folder name", file_path);

	/* Does the containing folder exist? */
	if (gitfo_isdir(target_folder_path)) {
		git__joinpath(target_folder_path, target_folder_path, ""); /* Ensure there's a trailing slash */

		/* Let's create the tree structure */
		error = gitfo_mkdir_recurs(target_folder_path, mode);
		if (error < GIT_SUCCESS)
			return error;	/* The callee already takes care of setting the correct error message. */
	}

	return GIT_SUCCESS;
}

int gitfo_mktemp(char *path_out, const char *filename)
{
	int fd;

	strcpy(path_out, filename);
	strcat(path_out, "_git2_XXXXXX");

#if defined(_MSC_VER)
	/* FIXME: there may be race conditions when multi-threading
	 * with the library */
	if (_mktemp_s(path_out, GIT_PATH_MAX) != 0)
		return git__throw(GIT_EOSERR, "Failed to make temporary file %s", path_out);

	fd = gitfo_creat(path_out, 0744);
#else
	fd = mkstemp(path_out);
#endif

	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to create temporary file %s", path_out);
}

int gitfo_open(const char *path, int flags)
{
	int fd = open(path, flags | O_BINARY);
	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to open %s", path);
}

int gitfo_creat(const char *path, int mode)
{
	int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, mode);
	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to create file. Could not open %s", path);
}

int gitfo_creat_force(const char *path, int mode)
{
	if (gitfo_mkdir_2file(path) < GIT_SUCCESS)
		return git__throw(GIT_EOSERR, "Failed to create file %s", path);

	return gitfo_creat(path, mode);
}

int gitfo_creat_locked(const char *path, int mode)
{
	int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_EXCL, mode);
	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to create locked file. Could not open %s", path);
}

int gitfo_creat_locked_force(const char *path, int mode)
{
	if (gitfo_mkdir_2file(path) < GIT_SUCCESS)
		return git__throw(GIT_EOSERR, "Failed to create locked file %s", path);

	return gitfo_creat_locked(path, mode);
}

int gitfo_read(git_file fd, void *buf, size_t cnt)
{
	char *b = buf;
	while (cnt) {
		ssize_t r = read(fd, b, cnt);
		if (r < 0) {
			if (errno == EINTR || errno == EAGAIN)
				continue;
			return git__throw(GIT_EOSERR, "Failed to read from file");
		}
		if (!r) {
			errno = EPIPE;
			return git__throw(GIT_EOSERR, "Failed to read from file");
		}
		cnt -= r;
		b += r;
	}
	return GIT_SUCCESS;
}

int gitfo_write(git_file fd, void *buf, size_t cnt)
{
	char *b = buf;
	while (cnt) {
		ssize_t r = write(fd, b, cnt);
		if (r < 0) {
			if (errno == EINTR || errno == EAGAIN)
				continue;
			return git__throw(GIT_EOSERR, "Failed to write to file");
		}
		if (!r) {
			errno = EPIPE;
			return git__throw(GIT_EOSERR, "Failed to write to file");
		}
		cnt -= r;
		b += r;
	}
	return GIT_SUCCESS;
}

int gitfo_isdir(const char *path)
{
	struct stat st;
	int len, stat_error;

	if (!path)
		return git__throw(GIT_ENOTFOUND, "No path given to gitfo_isdir");

	len = strlen(path);

	/* win32: stat path for folders cannot end in a slash */
	if (path[len - 1] == '/') {
		char *path_fixed = NULL;
		path_fixed = git__strdup(path);
		path_fixed[len - 1] = 0;
		stat_error = gitfo_stat(path_fixed, &st);
		free(path_fixed);
	} else {
		stat_error = gitfo_stat(path, &st);
	}

	if (stat_error < GIT_SUCCESS)
		return git__throw(GIT_ENOTFOUND, "%s does not exist", path);

	if (!S_ISDIR(st.st_mode))
		return git__throw(GIT_ENOTFOUND, "%s is not a directory", path);

	return GIT_SUCCESS;
}

int gitfo_isfile(const char *path)
{
	struct stat st;
	int stat_error;

	if (!path)
		return git__throw(GIT_ENOTFOUND, "No path given to gitfo_isfile");

	stat_error = gitfo_stat(path, &st);

	if (stat_error < GIT_SUCCESS)
		return git__throw(GIT_ENOTFOUND, "%s does not exist", path);

	if (!S_ISREG(st.st_mode))
		return git__throw(GIT_ENOTFOUND, "%s is not a file", path);

	return GIT_SUCCESS;
}

int gitfo_exists(const char *path)
{
	assert(path);
	return access(path, F_OK);
}

git_off_t gitfo_size(git_file fd)
{
	struct stat sb;
	if (gitfo_fstat(fd, &sb))
		return git__throw(GIT_EOSERR, "Failed to get size of file. File missing or corrupted");
	return sb.st_size;
}

int gitfo_read_file(gitfo_buf *obj, const char *path)
{
	git_file fd;
	size_t len;
	git_off_t size;
	unsigned char *buff;

	assert(obj && path && *path);

	if ((fd = gitfo_open(path, O_RDONLY)) < 0)
		return git__throw(GIT_ERROR, "Failed to open %s for reading", path);

	if (((size = gitfo_size(fd)) < 0) || !git__is_sizet(size+1)) {
		gitfo_close(fd);
		return git__throw(GIT_ERROR, "Failed to read file `%s`. An error occured while calculating its size", path);
	}
	len = (size_t) size;

	if ((buff = git__malloc(len + 1)) == NULL) {
		gitfo_close(fd);
		return GIT_ENOMEM;
	}

	if (gitfo_read(fd, buff, len) < 0) {
		gitfo_close(fd);
		free(buff);
		return git__throw(GIT_ERROR, "Failed to read file `%s`", path);
	}
	buff[len] = '\0';

	gitfo_close(fd);

	obj->data = buff;
	obj->len  = len;

	return GIT_SUCCESS;
}

void gitfo_free_buf(gitfo_buf *obj)
{
	assert(obj);
	free(obj->data);
	obj->data = NULL;
}

int gitfo_mv(const char *from, const char *to)
{
	int error;

#ifdef GIT_WIN32
	/*
	 * Win32 POSIX compilance my ass. If the destination
	 * file exists, the `rename` call fails. This is as
	 * close as it gets with the Win32 API.
	 */
	error = MoveFileEx(from, to, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) ? GIT_SUCCESS : GIT_EOSERR;
#else
	/* Don't even try this on Win32 */
	if (!link(from, to)) {
		gitfo_unlink(from);
		return GIT_SUCCESS;
	}

	if (!rename(from, to))
		return GIT_SUCCESS;

	error = GIT_EOSERR;
#endif

	if (error < GIT_SUCCESS)
		return git__throw(error, "Failed to move file from `%s` to `%s`", from, to);

	return GIT_SUCCESS;
}

int gitfo_mv_force(const char *from, const char *to)
{
	if (gitfo_mkdir_2file(to) < GIT_SUCCESS)
		return GIT_EOSERR;	/* The callee already takes care of setting the correct error message. */

	return gitfo_mv(from, to);	/* The callee already takes care of setting the correct error message. */
}

int gitfo_map_ro(git_map *out, git_file fd, git_off_t begin, size_t len)
{
	if (git__mmap(out, len, GIT_PROT_READ, GIT_MAP_SHARED, fd, begin) < GIT_SUCCESS)
		return GIT_EOSERR;
	return GIT_SUCCESS;
}

void gitfo_free_map(git_map *out)
{
	git__munmap(out);
}

int gitfo_dirent(
	char *path,
	size_t path_sz,
	int (*fn)(void *, char *),
	void *arg)
{
	size_t wd_len = strlen(path);
	DIR *dir;
	struct dirent *de;

	if (!wd_len || path_sz < wd_len + 2)
		return git__throw(GIT_EINVALIDARGS, "Failed to process `%s` tree structure. Path is either empty or buffer size is too short", path);

	while (path[wd_len - 1] == '/')
		wd_len--;
	path[wd_len++] = '/';
	path[wd_len] = '\0';

	dir = opendir(path);
	if (!dir)
		return git__throw(GIT_EOSERR, "Failed to process `%s` tree structure. An error occured while opening the directory", path);

	while ((de = readdir(dir)) != NULL) {
		size_t de_len;
		int result;

		/* always skip '.' and '..' */
		if (de->d_name[0] == '.') {
			if (de->d_name[1] == '\0')
				continue;
			if (de->d_name[1] == '.' && de->d_name[2] == '\0')
				continue;
		}

		de_len = strlen(de->d_name);
		if (path_sz < wd_len + de_len + 1) {
			closedir(dir);
			return git__throw(GIT_ERROR, "Failed to process `%s` tree structure. Buffer size is too short", path);
		}

		strcpy(path + wd_len, de->d_name);
		result = fn(arg, path);
		if (result < GIT_SUCCESS) {
			closedir(dir);
			return result;	/* The callee is reponsible for setting the correct error message */
		}
		if (result > 0) {
			closedir(dir);
			return result;
		}
	}

	closedir(dir);
	return GIT_SUCCESS;
}

void gitfo_posixify_path(char *path)
{
	#if GIT_PLATFORM_PATH_SEP != '/'
		while (*path) {
			if (*path == GIT_PLATFORM_PATH_SEP)
				*path = '/';

			path++;
		}
	#endif
}

int gitfo_retrieve_path_root_offset(const char *path)
{
	int offset = 0;

#ifdef GIT_WIN32

	/* Does the root of the path look like a windows drive ? */
	if (isalpha(path[0]) && (path[1] == ':'))
		offset += 2;

#endif

	if (*(path + offset) == '/')
		return offset;

	return -1;	/* Not a real error. Rather a signal than the path is not rooted */
}

int gitfo_mkdir_recurs(const char *path, int mode)
{
	int error, root_path_offset;
	char *pp, *sp;
    char *path_copy = git__strdup(path);

	if (path_copy == NULL)
		return GIT_ENOMEM;

	error = GIT_SUCCESS;
	pp = path_copy;

	root_path_offset = gitfo_retrieve_path_root_offset(pp);
	if (root_path_offset > 0)
		pp += root_path_offset; /* On Windows, will skip the drive name (eg. C: or D:) */

    while (error == GIT_SUCCESS && (sp = strchr(pp, '/')) != NULL) {
		if (sp != pp && gitfo_isdir(path_copy) < GIT_SUCCESS) {
			*sp = 0;
			error = gitfo_mkdir(path_copy, mode);

			/* Do not choke while trying to recreate an existing directory */
			if (errno == EEXIST)
				error = GIT_SUCCESS;

			*sp = '/';
		}

		pp = sp + 1;
	}

	if (*pp != '\0' && error == GIT_SUCCESS) {
		error = gitfo_mkdir(path, mode);
		if (errno == EEXIST)
			error = GIT_SUCCESS;
	}

	free(path_copy);

	if (error < GIT_SUCCESS)
		return git__throw(error, "Failed to recursively create `%s` tree structure", path);

	return GIT_SUCCESS;
}

static int retrieve_previous_path_component_start(const char *path)
{
	int offset, len, root_offset, start = 0;

	root_offset = gitfo_retrieve_path_root_offset(path);
	if (root_offset > -1)
{'fraction_non_alphanumeric': 0.13547896604156107, 'fraction_numerical': 0.006842372022301064, 'mean_word_length': 4.039846743295019, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '174608', 'n_tokens_mistral': 8256, 'n_tokens_neox': 7701, 'n_words': 2151}