<commit_before>/*
   Copyright 2010 <NAME>

   This file is part of Tundra.

   Tundra is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   Tundra is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with Tundra.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "relcache.h"
#include "engine.h"
#include "util.h"
#include "portable.h"

#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>

#define TD_RELCACHE_FILE ".tundra-relcache"

enum {
	TD_RELCACHE_TTL_DAYS = 7,
	TD_RELCACHE_TTL_SECS = 60 * 60 * 24 * TD_RELCACHE_TTL_DAYS,
};

static const uint32_t td_relcache_magic =
	0xffed0000u + (sizeof(void*) << 8) + (sizeof(long) << 4) + sizeof(time_t);

/*
 * Frozen file format for relation cache entries.
 *
 * This format is platform-specific and the resulting files cannot be moved
 * between platforms. Even if the format would allow moving it wouldn't make
 * sense anyway as it contains platform-specific paths. If multiple Tundra
 * builds want to access the same build directory, we should salt the filename
 * instead so each build configuration gets a unique file.
 *
 * - header (td_frozen_rel_header)
 *
 * - string block (header.string_block_size bytes)
 *
 *   This block contains null-terminated strings, back to back that are indexed
 *   from relation entries. Becaure they are null-terminated, the entire block
 *   can be made resident and pointers set up straight into the block.
 *
 * - relation block (header.relation_count * sizeof(unsigned int) bytes)
 *
 *   This block contains string block indices for children; e.g. if node 0 has
 *   to children, 80 and 90 and node 1 has one child, 400, the first three
 *   indices of this block will be 80, 90, 400. Later when reading the nodes
 *   each node encodes an index and range into this block to indicate what
 *   files are related. The purpose of this whole affair is to avoid having
 *   variable length reads for each node. We want to do as few read calls as
 *   possible.
 *
 * - node block (header.cell_count)
 *
 */

typedef struct td_frozen_rel_header {
	uint32_t magic;
	uint32_t string_block_size;
	uint32_t relation_count;
	uint32_t node_count;
} td_frozen_rel_header;

typedef struct td_frozen_relation {
	uint32_t string_index;
	uint32_t salt;
	time_t access_time;
	uint32_t first_relation_offset;
	uint32_t relation_count;
	td_digest signature;
} td_frozen_relation;

typedef struct td_frozen_reldata {
	td_frozen_rel_header header;
	char *string_block;
	uint32_t *index_block;
	td_frozen_relation *relations;
} td_frozen_reldata;

/* Caches a relation between a file and set of other files (such as set of
 * included files) */
typedef struct td_relcell
{
	/* source file */
	struct td_file *file;

	/* a salt value to make this relation unique */
	uint32_t salt;

	/* the related files */
	int count;
	struct td_file **files;

	/* a timestamp when this node was last touched, for gc */
	time_t timestamp;

	uint32_t child_list_start;

	/* the signature of the file when this relation was cached */
	td_digest signature;

	/* for hash table linked list maintenance */
	struct td_relcell *bucket_next;
} td_relcell;

td_file **
td_engine_get_relations(td_engine *engine, td_file *file, unsigned int salt, int *count_out)
{
	td_relcell *chain;
	int bucket;
	unsigned int hash;

	hash = file->hash ^ salt;
	bucket = hash % engine->relhash_size;
	assert(bucket >= 0 && bucket < engine->relhash_size);
	chain = engine->relhash[bucket];

	while (chain)
	{
		if (salt == chain->salt && file == chain->file)
		{
			/* Check that the cached file signature still is valid. This
			 * prevents using stale cached information for generated header
			 * files. */
			const td_digest *signature = td_get_signature(engine, chain->file);

			if (0 != memcmp(&chain->signature, signature, sizeof(td_digest)))
				goto leave;

			*count_out = chain->count;
			return chain->files;
		}
		chain = chain->bucket_next;
	}

leave:
	*count_out = 0;
	return NULL;
}

static void
populate_relcell(
		td_engine *engine,
		td_relcell* cell,
		td_file *file,
		unsigned int salt,
		int count,
		td_file **files,
		const td_digest *sig)
{
	size_t memsize = sizeof(td_file*) * count;
	cell->file = file;
	cell->salt = salt;
	cell->count = count;
	cell->files = (td_file **) td_page_alloc(&engine->alloc, memsize);
	cell->timestamp = engine->start_time;

	memcpy(&cell->signature, sig, sizeof(td_digest));

	memcpy(&cell->files[0], files, memsize);
}

void
set_relations(td_engine *engine, td_file *file, unsigned int salt, int count, td_file **files, const td_digest* digest)
{
	unsigned int hash;
	td_relcell *chain;
	int bucket;

	hash = file->hash ^ salt;
	bucket = hash % engine->relhash_size;
	chain = engine->relhash[bucket];

	while (chain)
	{
		if (salt == chain->salt && file == chain->file)
		{
			populate_relcell(engine, chain, file, salt, count, files, digest);
			return;
		}

		chain = chain->bucket_next;
	}

	++engine->stats.relation_count;
	chain = (td_relcell*) td_page_alloc(&engine->alloc, sizeof(td_relcell));
	populate_relcell(engine, chain, file, salt, count, files, digest);
	chain->bucket_next = engine->relhash[bucket];
	engine->relhash[bucket] = chain;
}

void
td_engine_set_relations(td_engine *engine, td_file *file, unsigned int salt, int count, td_file **files)
{
	set_relations(engine, file, salt, count, files, td_get_signature(engine, file));
}

static void
persist_filename(FILE *rcf, td_file *file, uint32_t *cursor)
{
	if (~0u == file->frozen_relstring_index)
	{<commit_msg>Fixed relation cache was not thread safe.<commit_after>/*
   Copyright 2010 <NAME>

   This file is part of Tundra.

   Tundra is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   Tundra is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with Tundra.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "relcache.h"
#include "engine.h"
#include "util.h"
#include "portable.h"

#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>

#define TD_RELCACHE_FILE ".tundra-relcache"

enum {
	TD_RELCACHE_TTL_DAYS = 7,
	TD_RELCACHE_TTL_SECS = 60 * 60 * 24 * TD_RELCACHE_TTL_DAYS,
};

static const uint32_t td_relcache_magic =
	0xffed0000u + (sizeof(void*) << 8) + (sizeof(long) << 4) + sizeof(time_t);

/*
 * Frozen file format for relation cache entries.
 *
 * This format is platform-specific and the resulting files cannot be moved
 * between platforms. Even if the format would allow moving it wouldn't make
 * sense anyway as it contains platform-specific paths. If multiple Tundra
 * builds want to access the same build directory, we should salt the filename
 * instead so each build configuration gets a unique file.
 *
 * - header (td_frozen_rel_header)
 *
 * - string block (header.string_block_size bytes)
 *
 *   This block contains null-terminated strings, back to back that are indexed
 *   from relation entries. Becaure they are null-terminated, the entire block
 *   can be made resident and pointers set up straight into the block.
 *
 * - relation block (header.relation_count * sizeof(unsigned int) bytes)
 *
 *   This block contains string block indices for children; e.g. if node 0 has
 *   to children, 80 and 90 and node 1 has one child, 400, the first three
 *   indices of this block will be 80, 90, 400. Later when reading the nodes
 *   each node encodes an index and range into this block to indicate what
 *   files are related. The purpose of this whole affair is to avoid having
 *   variable length reads for each node. We want to do as few read calls as
 *   possible.
 *
 * - node block (header.cell_count)
 *
 */

typedef struct td_frozen_rel_header {
	uint32_t magic;
	uint32_t string_block_size;
	uint32_t relation_count;
	uint32_t node_count;
} td_frozen_rel_header;

typedef struct td_frozen_relation {
	uint32_t string_index;
	uint32_t salt;
	time_t access_time;
	uint32_t first_relation_offset;
	uint32_t relation_count;
	td_digest signature;
} td_frozen_relation;

typedef struct td_frozen_reldata {
	td_frozen_rel_header header;
	char *string_block;
	uint32_t *index_block;
	td_frozen_relation *relations;
} td_frozen_reldata;

/* Caches a relation between a file and set of other files (such as set of
 * included files) */
typedef struct td_relcell
{
	/* source file */
	struct td_file *file;

	/* a salt value to make this relation unique */
	uint32_t salt;

	/* the related files */
	int count;
	struct td_file **files;

	/* a timestamp when this node was last touched, for gc */
	time_t timestamp;

	uint32_t child_list_start;

	/* the signature of the file when this relation was cached */
	td_digest signature;

	/* for hash table linked list maintenance */
	struct td_relcell *bucket_next;
} td_relcell;

td_file **
td_engine_get_relations(td_engine *engine, td_file *file, unsigned int salt, int *count_out)
{
	td_relcell *chain;
	int bucket;
	int count_result = 0;
	td_file **result = NULL;
	unsigned int hash;

	td_mutex_lock_or_die(engine->lock);

	hash = file->hash ^ salt;
	bucket = hash % engine->relhash_size;
	assert(bucket >= 0 && bucket < engine->relhash_size);
	chain = engine->relhash[bucket];

	while (chain)
	{
		if (salt == chain->salt && file == chain->file)
		{
			const td_digest *signature = td_get_signature(engine, chain->file);

			/* Check that the cached file signature still is valid. This
			 * prevents using stale cached information for generated header
			 * files. */
			if (0 == memcmp(&chain->signature, signature, sizeof(td_digest)))
			{
				count_result = chain->count;
				result = chain->files;
			}

			break;
		}
		chain = chain->bucket_next;
	}

	td_mutex_unlock_or_die(engine->lock);
	*count_out = count_result;
	return result;
}

static void
populate_relcell(
		td_engine *engine,
		td_relcell* cell,
		td_file *file,
		unsigned int salt,
		int count,
		td_file **files,
		const td_digest *sig)
{
	size_t memsize = sizeof(td_file*) * count;
	cell->file = file;
	cell->salt = salt;
	cell->count = count;
	cell->files = (td_file **) td_page_alloc(&engine->alloc, memsize);
	cell->timestamp = engine->start_time;

	memcpy(&cell->signature, sig, sizeof(td_digest));

	memcpy(&cell->files[0], files, memsize);
}

void
set_relations(td_engine *engine, td_file *file, unsigned int salt, int count, td_file **files, const td_digest* digest)
{
	unsigned int hash;
	td_relcell *chain;
	int bucket;

	hash = file->hash ^ salt;
	bucket = hash % engine->relhash_size;
	chain = engine->relhash[bucket];

	while (chain)
	{
		if (salt == chain->salt && file == chain->file)
		{
			populate_relcell(engine, chain, file, salt, count, files, digest);
			return;
		}

		chain = chain->bucket_next;
	}

	++engine->stats.relation_count;
	chain = (td_relcell*) td_page_alloc(&engine->alloc, sizeof(td_relcell));
	populate_relcell(engine, chain, file, salt, count, files, digest);
	chain->bucket_next = engine->relhash[bucket];
	engine->relhash[bucket] = chain;
}

void
td_engine_set_relations(td_engine *engine, td_file *file, unsigned int salt, int count, td_file **files)
{
	td_mutex_lock_or_die(engine->lock);
	set_relations(engine, file, salt, count, files, td_get_signature(engine, file));
	td_mutex_unlock_or_die(engine->lock);
}

static void
persist_filename(FILE *rcf, td_file *file, uint32_t *cursor)
{
	if (~0u == file->frozen_relstring_index)
	{
{'fraction_non_alphanumeric': 0.09731598880289807, 'fraction_numerical': 0.010867775399308414, 'mean_word_length': 4.1711366538952745, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4373334', 'n_tokens_mistral': 4210, 'n_tokens_neox': 3912, 'n_words': 1352}