<commit_before>/*
 * Copyright (C) the libgit2 contributors. All rights reserved.
 *
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
 */

#include "common.h"
#include "odb.h"
#include "pack.h"
#include "delta-apply.h"
#include "sha1_lookup.h"
#include "mwindow.h"
#include "fileops.h"

#include "git2/oid.h"
#include <zlib.h>

static int packfile_open(struct git_pack_file *p);
static git_off_t nth_packed_object_offset(const struct git_pack_file *p, uint32_t n);
int packfile_unpack_compressed(
		git_rawobj *obj,
		struct git_pack_file *p,
		git_mwindow **w_curs,
		git_off_t *curpos,
		size_t size,
		git_otype type);

/* Can find the offset of an object given
 * a prefix of an identifier.
 * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid
 * is ambiguous within the pack.
 * This method assumes that len is between
 * GIT_OID_MINPREFIXLEN and GIT_OID_HEXSZ.
 */
static int pack_entry_find_offset(
		git_off_t *offset_out,
		git_oid *found_oid,
		struct git_pack_file *p,
		const git_oid *short_oid,
		size_t len);

static int packfile_error(const char *message)
{
	giterr_set(GITERR_ODB, "Invalid pack file - %s", message);
	return -1;
}

/********************
 * Delta base cache
 ********************/

static git_pack_cache_entry *new_cache_object(git_rawobj *source)
{
	git_pack_cache_entry *e = git__calloc(1, sizeof(git_pack_cache_entry));
	if (!e)
		return NULL;

	memcpy(&e->raw, source, sizeof(git_rawobj));

	return e;
}

static void free_cache_object(void *o)
{
	git_pack_cache_entry *e = (git_pack_cache_entry *)o;

	if (e != NULL) {
		assert(e->refcount.val == 0);
		git__free(e->raw.data);
		git__free(e);
	}
}

static void cache_free(git_pack_cache *cache)
{
	khiter_t k;

	if (cache->entries) {
		for (k = kh_begin(cache->entries); k != kh_end(cache->entries); k++) {
			if (kh_exist(cache->entries, k))
				free_cache_object(kh_value(cache->entries, k));
		}

		git_offmap_free(cache->entries);
	}
}

static int cache_init(git_pack_cache *cache)
{
	memset(cache, 0, sizeof(git_pack_cache));
	cache->entries = git_offmap_alloc();
	GITERR_CHECK_ALLOC(cache->entries);
	cache->memory_limit = GIT_PACK_CACHE_MEMORY_LIMIT;
	git_mutex_init(&cache->lock);

	return 0;
}

static git_pack_cache_entry *cache_get(git_pack_cache *cache, size_t offset)
{
	khiter_t k;
	git_pack_cache_entry *entry = NULL;

	git_mutex_lock(&cache->lock);
	k = kh_get(off, cache->entries, offset);
	if (k != kh_end(cache->entries)) { /* found it */
		entry = kh_value(cache->entries, k);
		git_atomic_inc(&entry->refcount);
		entry->last_usage = cache->use_ctr++;
	}
	git_mutex_unlock(&cache->lock);

	return entry;
}

/* Run with the cache lock held */
static void free_lowest_entry(git_pack_cache *cache)
{
	git_pack_cache_entry *lowest = NULL, *entry;
	khiter_t k, lowest_k;

	for (k = kh_begin(cache->entries); k != kh_end(cache->entries); k++) {
		if (!kh_exist(cache->entries, k))
			continue;

		entry = kh_value(cache->entries, k);
		if (lowest == NULL || entry->last_usage < lowest->last_usage) {
			lowest_k = k;
			lowest = entry;
		}
	}

	if (!lowest) /* there's nothing to free */
		return;

	cache->memory_used -= lowest->raw.len;
	kh_del(off, cache->entries, lowest_k);
	free_cache_object(lowest);
}

static int cache_add(git_pack_cache *cache, git_rawobj *base, git_off_t offset)
{
	git_pack_cache_entry *entry;
	int error, exists = 0;
	khiter_t k;

	if (base->len > GIT_PACK_CACHE_SIZE_LIMIT)
		return -1;

	entry = new_cache_object(base);
	if (entry) {
		git_mutex_lock(&cache->lock);
		/* Add it to the cache if nobody else has */
		exists = kh_get(off, cache->entries, offset) != kh_end(cache->entries);
		if (!exists) {
			while (cache->memory_used + base->len > cache->memory_limit)
				free_lowest_entry(cache);

			k = kh_put(off, cache->entries, offset, &error);
			assert(error != 0);
			kh_value(cache->entries, k) = entry;
			cache->memory_used += entry->raw.len;
		}
		git_mutex_unlock(&cache->lock);
		/* Somebody beat us to adding it into the cache */
		if (exists) {
			git__free(entry);
			return -1;
		}
	}

	return 0;
}

/***********************************************************
 *
 * PACK INDEX METHODS
 *
 ***********************************************************/<commit_msg>pack: fixes to the cache<commit_after>/*
 * Copyright (C) the libgit2 contributors. All rights reserved.
 *
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
 */

#include "common.h"
#include "odb.h"
#include "pack.h"
#include "delta-apply.h"
#include "sha1_lookup.h"
#include "mwindow.h"
#include "fileops.h"

#include "git2/oid.h"
#include <zlib.h>

static int packfile_open(struct git_pack_file *p);
static git_off_t nth_packed_object_offset(const struct git_pack_file *p, uint32_t n);
int packfile_unpack_compressed(
		git_rawobj *obj,
		struct git_pack_file *p,
		git_mwindow **w_curs,
		git_off_t *curpos,
		size_t size,
		git_otype type);

/* Can find the offset of an object given
 * a prefix of an identifier.
 * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid
 * is ambiguous within the pack.
 * This method assumes that len is between
 * GIT_OID_MINPREFIXLEN and GIT_OID_HEXSZ.
 */
static int pack_entry_find_offset(
		git_off_t *offset_out,
		git_oid *found_oid,
		struct git_pack_file *p,
		const git_oid *short_oid,
		size_t len);

static int packfile_error(const char *message)
{
	giterr_set(GITERR_ODB, "Invalid pack file - %s", message);
	return -1;
}

/********************
 * Delta base cache
 ********************/

static git_pack_cache_entry *new_cache_object(git_rawobj *source)
{
	git_pack_cache_entry *e = git__calloc(1, sizeof(git_pack_cache_entry));
	if (!e)
		return NULL;

	memcpy(&e->raw, source, sizeof(git_rawobj));

	return e;
}

static void free_cache_object(void *o)
{
	git_pack_cache_entry *e = (git_pack_cache_entry *)o;

	if (e != NULL) {
		assert(e->refcount.val == 0);
		git__free(e->raw.data);
		git__free(e);
	}
}

static void cache_free(git_pack_cache *cache)
{
	khiter_t k;

	if (cache->entries) {
		for (k = kh_begin(cache->entries); k != kh_end(cache->entries); k++) {
			if (kh_exist(cache->entries, k))
				free_cache_object(kh_value(cache->entries, k));
		}

		git_offmap_free(cache->entries);
	}
}

static int cache_init(git_pack_cache *cache)
{
	memset(cache, 0, sizeof(git_pack_cache));
	cache->entries = git_offmap_alloc();
	GITERR_CHECK_ALLOC(cache->entries);
	cache->memory_limit = GIT_PACK_CACHE_MEMORY_LIMIT;
	git_mutex_init(&cache->lock);

	return 0;
}

static git_pack_cache_entry *cache_get(git_pack_cache *cache, git_off_t offset)
{
	khiter_t k;
	git_pack_cache_entry *entry = NULL;

	if (git_mutex_lock(&cache->lock) < 0)
		return NULL;

	k = kh_get(off, cache->entries, offset);
	if (k != kh_end(cache->entries)) { /* found it */
		entry = kh_value(cache->entries, k);
		git_atomic_inc(&entry->refcount);
		entry->last_usage = cache->use_ctr++;
	}
	git_mutex_unlock(&cache->lock);

	return entry;
}

/* Run with the cache lock held */
static void free_lowest_entry(git_pack_cache *cache)
{
	git_pack_cache_entry *lowest = NULL, *entry;
	khiter_t k, lowest_k;

	for (k = kh_begin(cache->entries); k != kh_end(cache->entries); k++) {
		if (!kh_exist(cache->entries, k))
			continue;

		entry = kh_value(cache->entries, k);
		if (lowest == NULL || entry->last_usage < lowest->last_usage) {
			lowest_k = k;
			lowest = entry;
		}
	}

	if (!lowest) /* there's nothing to free */
		return;

	cache->memory_used -= lowest->raw.len;
	kh_del(off, cache->entries, lowest_k);
	free_cache_object(lowest);
}

static int cache_add(git_pack_cache *cache, git_rawobj *base, git_off_t offset)
{
	git_pack_cache_entry *entry;
	int error, exists = 0;
	khiter_t k;

	if (base->len > GIT_PACK_CACHE_SIZE_LIMIT)
		return -1;

	entry = new_cache_object(base);
	if (entry) {
		if (git_mutex_lock(&cache->lock) < 0) {
			giterr_set(GITERR_OS, "failed to lock cache");
			return -1;
		}
		/* Add it to the cache if nobody else has */
		exists = kh_get(off, cache->entries, offset) != kh_end(cache->entries);
		if (!exists) {
			while (cache->memory_used + base->len > cache->memory_limit)
				free_lowest_entry(cache);

			k = kh_put(off, cache->entries, offset, &error);
			assert(error != 0);
			kh_value(cache->entries, k) = entry;
			cache->memory_used += entry->raw.len;
		}
		git_mutex_unlock(&cache->lock);
		/* Somebody beat us to adding it into the cache */
		if (exists) {
			git__free(entry);
			return -1;
		}
	}

	return 0;
}

/***********************************************************
 *
 * PACK INDEX METHODS
 *
 ***********************************************************/
{'fraction_non_alphanumeric': 0.1657103511590544, 'fraction_numerical': 0.0042460408537984854, 'mean_word_length': 4.622580645161291, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1239492', 'n_tokens_mistral': 3452, 'n_tokens_neox': 3231, 'n_words': 759}