<commit_before>#include "common.h"
#include "repository.h"
#include "filebuf.h"
#include "attr_file.h"
#include "attrcache.h"
#include "git2/blob.h"
#include "git2/tree.h"
#include "index.h"
#include <ctype.h>

static void attr_file_free(git_attr_file *file)
{
	bool unlock = !git_mutex_lock(&file->lock);
	git_attr_file__clear_rules(file, false);
	git_pool_clear(&file->pool);
	if (unlock)
		git_mutex_unlock(&file->lock);
	git_mutex_free(&file->lock);

	git__memzero(file, sizeof(*file));
	git__free(file);
}

int git_attr_file__new(
	git_attr_file **out,
	git_attr_file_entry *entry,
	git_attr_file_source source)
{
	git_attr_file *attrs = git__calloc(1, sizeof(git_attr_file));
	GITERR_CHECK_ALLOC(attrs);

	if (git_mutex_init(&attrs->lock) < 0) {
		giterr_set(GITERR_OS, "Failed to initialize lock");
		git__free(attrs);
		return -1;
	}

	if (git_pool_init(&attrs->pool, 1, 0) < 0) {
		attr_file_free(attrs);
		return -1;
	}

	GIT_REFCOUNT_INC(attrs);
	attrs->entry  = entry;
	attrs->source = source;
	*out = attrs;
	return 0;
}

int git_attr_file__clear_rules(git_attr_file *file, bool need_lock)
{
	unsigned int i;
	git_attr_rule *rule;

	if (need_lock && git_mutex_lock(&file->lock) < 0) {
		giterr_set(GITERR_OS, "Failed to lock attribute file");
		return -1;
	}

	git_vector_foreach(&file->rules, i, rule)
		git_attr_rule__free(rule);
	git_vector_free(&file->rules);

	if (need_lock)
		git_mutex_unlock(&file->lock);

	return 0;
}

void git_attr_file__free(git_attr_file *file)
{
	if (!file)
		return;
	GIT_REFCOUNT_DEC(file, attr_file_free);
}

static int attr_file_oid_from_index(
	git_oid *oid, git_repository *repo, const char *path)
{
	int error;
	git_index *idx;
	size_t pos;
	const git_index_entry *entry;

	if ((error = git_repository_index__weakptr(&idx, repo)) < 0 ||
		(error = git_index__find_pos(&pos, idx, path, 0, 0)) < 0)
		return error;

	if (!(entry = git_index_get_byindex(idx, pos)))
		return GIT_ENOTFOUND;

	*oid = entry->id;
	return 0;
}

int git_attr_file__load(
	git_attr_file **out,
	git_repository *repo,
	git_attr_file_entry *entry,
	git_attr_file_source source,
	git_attr_file_parser parser)
{
	int error = 0;
	git_blob *blob = NULL;
	git_buf content = GIT_BUF_INIT;
	const char *data = NULL;
	git_attr_file *file;
	struct stat st;

	*out = NULL;

	switch (source) {
	case GIT_ATTR_FILE__IN_MEMORY:
		/* in-memory attribute file doesn't need data */
		break;
	case GIT_ATTR_FILE__FROM_INDEX: {
		git_oid id;

		if ((error = attr_file_oid_from_index(&id, repo, entry->path)) < 0 ||
			(error = git_blob_lookup(&blob, repo, &id)) < 0)
			return error;

		data = git_blob_rawcontent(blob);
		break;
	}
	case GIT_ATTR_FILE__FROM_FILE: {
		int fd;

		if (p_stat(entry->fullpath, &st) < 0)
			return git_path_set_error(errno, entry->fullpath, "stat");
		if (S_ISDIR(st.st_mode))
			return GIT_ENOTFOUND;

		/* For open or read errors, return ENOTFOUND to skip item */
		/* TODO: issue warning when warning API is available */

		if ((fd = git_futils_open_ro(entry->fullpath)) < 0)
			return GIT_ENOTFOUND;

		error = git_futils_readbuffer_fd(&content, fd, (size_t)st.st_size);
		p_close(fd);

		if (error < 0)
			return GIT_ENOTFOUND;

		data = content.ptr;
		break;
	}
	default:
		giterr_set(GITERR_INVALID, "Unknown file source %d", source);
		return -1;
	}

	if ((error = git_attr_file__new(&file, entry, source)) < 0)
		goto cleanup;

	if (parser && (error = parser(repo, file, data)) < 0) {
		git_attr_file__free(file);
		goto cleanup;
	}

	/* write cache breaker */
	if (source == GIT_ATTR_FILE__FROM_INDEX)
		git_oid_cpy(&file->cache_data.oid, git_blob_id(blob));
	else if (source == GIT_ATTR_FILE__FROM_FILE)
		git_futils_filestamp_set_from_stat(&file->cache_data.stamp, &st);
	/* else always cacheable */

	*out = file;<commit_msg>attr_file: Do not assume ODB data is NULL-terminated<commit_after>#include "common.h"
#include "repository.h"
#include "filebuf.h"
#include "attr_file.h"
#include "attrcache.h"
#include "git2/blob.h"
#include "git2/tree.h"
#include "index.h"
#include <ctype.h>

static void attr_file_free(git_attr_file *file)
{
	bool unlock = !git_mutex_lock(&file->lock);
	git_attr_file__clear_rules(file, false);
	git_pool_clear(&file->pool);
	if (unlock)
		git_mutex_unlock(&file->lock);
	git_mutex_free(&file->lock);

	git__memzero(file, sizeof(*file));
	git__free(file);
}

int git_attr_file__new(
	git_attr_file **out,
	git_attr_file_entry *entry,
	git_attr_file_source source)
{
	git_attr_file *attrs = git__calloc(1, sizeof(git_attr_file));
	GITERR_CHECK_ALLOC(attrs);

	if (git_mutex_init(&attrs->lock) < 0) {
		giterr_set(GITERR_OS, "Failed to initialize lock");
		git__free(attrs);
		return -1;
	}

	if (git_pool_init(&attrs->pool, 1, 0) < 0) {
		attr_file_free(attrs);
		return -1;
	}

	GIT_REFCOUNT_INC(attrs);
	attrs->entry  = entry;
	attrs->source = source;
	*out = attrs;
	return 0;
}

int git_attr_file__clear_rules(git_attr_file *file, bool need_lock)
{
	unsigned int i;
	git_attr_rule *rule;

	if (need_lock && git_mutex_lock(&file->lock) < 0) {
		giterr_set(GITERR_OS, "Failed to lock attribute file");
		return -1;
	}

	git_vector_foreach(&file->rules, i, rule)
		git_attr_rule__free(rule);
	git_vector_free(&file->rules);

	if (need_lock)
		git_mutex_unlock(&file->lock);

	return 0;
}

void git_attr_file__free(git_attr_file *file)
{
	if (!file)
		return;
	GIT_REFCOUNT_DEC(file, attr_file_free);
}

static int attr_file_oid_from_index(
	git_oid *oid, git_repository *repo, const char *path)
{
	int error;
	git_index *idx;
	size_t pos;
	const git_index_entry *entry;

	if ((error = git_repository_index__weakptr(&idx, repo)) < 0 ||
		(error = git_index__find_pos(&pos, idx, path, 0, 0)) < 0)
		return error;

	if (!(entry = git_index_get_byindex(idx, pos)))
		return GIT_ENOTFOUND;

	*oid = entry->id;
	return 0;
}

int git_attr_file__load(
	git_attr_file **out,
	git_repository *repo,
	git_attr_file_entry *entry,
	git_attr_file_source source,
	git_attr_file_parser parser)
{
	int error = 0;
	git_blob *blob = NULL;
	git_buf content = GIT_BUF_INIT;
	git_attr_file *file;
	struct stat st;

	*out = NULL;

	switch (source) {
	case GIT_ATTR_FILE__IN_MEMORY:
		/* in-memory attribute file doesn't need data */
		break;
	case GIT_ATTR_FILE__FROM_INDEX: {
		git_oid id;

		if ((error = attr_file_oid_from_index(&id, repo, entry->path)) < 0 ||
			(error = git_blob_lookup(&blob, repo, &id)) < 0)
			return error;

		/* Do not assume that data straight from the ODB is NULL-terminated;
		 * copy the contents of a file to a buffer to work on */
		git_buf_put(&content, git_blob_rawcontent(blob), git_blob_rawsize(blob));
		break;
	}
	case GIT_ATTR_FILE__FROM_FILE: {
		int fd;

		if (p_stat(entry->fullpath, &st) < 0)
			return git_path_set_error(errno, entry->fullpath, "stat");
		if (S_ISDIR(st.st_mode))
			return GIT_ENOTFOUND;

		/* For open or read errors, return ENOTFOUND to skip item */
		/* TODO: issue warning when warning API is available */

		if ((fd = git_futils_open_ro(entry->fullpath)) < 0)
			return GIT_ENOTFOUND;

		error = git_futils_readbuffer_fd(&content, fd, (size_t)st.st_size);
		p_close(fd);

		if (error < 0)
			return GIT_ENOTFOUND;

		break;
	}
	default:
		giterr_set(GITERR_INVALID, "Unknown file source %d", source);
		return -1;
	}

	if ((error = git_attr_file__new(&file, entry, source)) < 0)
		goto cleanup;

	if (parser && (error = parser(repo, file, git_buf_cstr(&content))) < 0) {
		git_attr_file__free(file);
		goto cleanup;
	}

	/* write cache breaker */
	if (source == GIT_ATTR_FILE__FROM_INDEX)
		git_oid_cpy(&file->cache_data.oid, git_blob_id(blob));
	else if (source == GIT_ATTR_FILE__FROM_FILE)
		git_futils_filestamp_set_from_stat(&file->cache_data.stamp, &st);
	/* else always cacheable */

	*out = file;
{'fraction_non_alphanumeric': 0.1325566343042071, 'fraction_numerical': 0.006990291262135922, 'mean_word_length': 4.893211289092296, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6910815', 'n_tokens_mistral': 3258, 'n_tokens_neox': 3064, 'n_words': 666}