<commit_before>#include <config.h>

#include <stdlib.h>
#include <sys/types.h>
#include <stdio.h>
#include <string.h>

#include <ht.h>
#include <hash.h>
#include <log.h>

int
ht_init(ht_t *ht, hash_t buckets, ht_hash_t hash, ht_match_t match,
	ht_delete_t delete)
{
	if((ht->ht_table = malloc(buckets * (sizeof(ht_record_t *) + 1))) == NULL) {
		log_warning("ht_init: malloc");
		return -1;
	}

	bzero(ht->ht_table, buckets * (sizeof(ht_record_t *) + 1));

	ht->ht_buckets = buckets;
	ht->ht_hash = hash;
	ht->ht_match = match;
	ht->ht_delete = delete;
	ht->ht_records = 0;
	ht->ht_collisions = 0;
	ht->ht_head = 0;

	return 0;
}


ht_t *
ht_create(hash_t buckets, ht_hash_t hash, ht_match_t match, ht_delete_t delete)
{
	ht_t *ht;

	if((ht = (ht_t *) malloc(sizeof(ht_t))) == NULL) {
		log_warning("ht_create: malloc");
		return NULL;
	}

	if(ht_init(ht, buckets, hash, match, delete)) {
		log_warning("ht_create: ht_init failed");
		return NULL;
	}

	return ht;
}


void
ht_clear(ht_t *ht)
{
	ht_record_t *record;
	ht_record_t *next;
	hash_t i;

	for(i = 0; i < ht->ht_buckets; ++i) {
		for(record = ht->ht_table[i]; record != NULL; record = next) {
			next = record->htr_next;
			if(ht->ht_delete) {
				ht->ht_delete(record->htr_data);
			}
			free(record);
		}
	}

	free(ht->ht_table);

	return;
}


void
ht_delete(ht_t *ht)
{
	ht_clear(ht);
	free(ht);

	return;
}


void *
ht_lookup(ht_t *ht, void *data)
{
	ht_record_t *record;
	hash_t bucket;

	bucket = ht->ht_hash(data) % ht->ht_buckets;

	if(ht->ht_table[bucket] == NULL) {
		return NULL;
	}

	for(record = ht->ht_table[bucket];
		record != NULL;
		record = record->htr_next)
	{
		if(ht->ht_match(data, record->htr_data)) {
			return record->htr_data;
		}
	}

	return NULL;
}


void
ht_start(ht_t *ht, ht_pos_t *pos)
{
	if(ht->ht_records)
	{
		pos->htp_bucket = ht->ht_head;
		pos->htp_record = ht->ht_table[ht->ht_head];
	}
	else
	{
		pos->htp_bucket = 0;
		pos->htp_record = NULL;
	}

	return;
}


int8_t
ht_insert(ht_t *ht, void *data)
{
	ht_record_t *record;
	hash_t bucket;
	float lf, cf;

	if(ht_lookup(ht, data) != NULL) {
		log_debug("ht_insert: duplicate entry");
		return -1;
	}

	bucket = ht->ht_hash(data) % ht->ht_buckets;

	if((record = malloc(sizeof (ht_record_t))) == NULL) {
		log_warning("ht_insert: malloc");
		return -1;
	}

	record->htr_data = data;
	record->htr_next =
		ht->ht_table[bucket] != NULL ? ht->ht_table[bucket] : NULL;

	ht->ht_table[bucket] = record;

	if(record->htr_next != NULL) {
		++ht->ht_collisions;
		
		/*
		 * First record in collision chain needs to be counted too.
		 */
		if (record->htr_next->htr_next == NULL)
		{
			++ht->ht_collisions;
		}
	}
	++ht->ht_records;

	if(bucket < ht->ht_head || ht->ht_head == 0) {
		ht->ht_head = bucket;
	}

	lf = HT_LOADFACTOR(ht);
	if (lf > 70)
	{
		log_error("ht_insert: load %.1f%", lf);
	}

	cf = HT_COLLFACTOR(ht);
	if (cf > 10)
	{
		log_error("ht_insert: collisions %.1f%", cf);
	}

	return 0;
}


void
ht_remove(ht_t *ht, void *data)
{
	ht_record_t *record;
	ht_record_t **prev;
	hash_t bucket;
	int collision = 0;

	bucket = ht->ht_hash(data) % ht->ht_buckets;

	if(ht->ht_table[bucket] == NULL) {
		log_debug("ht_remove: record not found");
		return;
	}
	<commit_msg>Hash table collision warning threshold increased<commit_after>#include <config.h>

#include <stdlib.h>
#include <sys/types.h>
#include <stdio.h>
#include <string.h>

#include <ht.h>
#include <hash.h>
#include <log.h>

int
ht_init(ht_t *ht, hash_t buckets, ht_hash_t hash, ht_match_t match,
	ht_delete_t delete)
{
	if((ht->ht_table = malloc(buckets * (sizeof(ht_record_t *) + 1))) == NULL) {
		log_warning("ht_init: malloc");
		return -1;
	}

	bzero(ht->ht_table, buckets * (sizeof(ht_record_t *) + 1));

	ht->ht_buckets = buckets;
	ht->ht_hash = hash;
	ht->ht_match = match;
	ht->ht_delete = delete;
	ht->ht_records = 0;
	ht->ht_collisions = 0;
	ht->ht_head = 0;

	return 0;
}


ht_t *
ht_create(hash_t buckets, ht_hash_t hash, ht_match_t match, ht_delete_t delete)
{
	ht_t *ht;

	if((ht = (ht_t *) malloc(sizeof(ht_t))) == NULL) {
		log_warning("ht_create: malloc");
		return NULL;
	}

	if(ht_init(ht, buckets, hash, match, delete)) {
		log_warning("ht_create: ht_init failed");
		return NULL;
	}

	return ht;
}


void
ht_clear(ht_t *ht)
{
	ht_record_t *record;
	ht_record_t *next;
	hash_t i;

	for(i = 0; i < ht->ht_buckets; ++i) {
		for(record = ht->ht_table[i]; record != NULL; record = next) {
			next = record->htr_next;
			if(ht->ht_delete) {
				ht->ht_delete(record->htr_data);
			}
			free(record);
		}
	}

	free(ht->ht_table);

	return;
}


void
ht_delete(ht_t *ht)
{
	ht_clear(ht);
	free(ht);

	return;
}


void *
ht_lookup(ht_t *ht, void *data)
{
	ht_record_t *record;
	hash_t bucket;

	bucket = ht->ht_hash(data) % ht->ht_buckets;

	if(ht->ht_table[bucket] == NULL) {
		return NULL;
	}

	for(record = ht->ht_table[bucket];
		record != NULL;
		record = record->htr_next)
	{
		if(ht->ht_match(data, record->htr_data)) {
			return record->htr_data;
		}
	}

	return NULL;
}


void
ht_start(ht_t *ht, ht_pos_t *pos)
{
	if(ht->ht_records)
	{
		pos->htp_bucket = ht->ht_head;
		pos->htp_record = ht->ht_table[ht->ht_head];
	}
	else
	{
		pos->htp_bucket = 0;
		pos->htp_record = NULL;
	}

	return;
}


int8_t
ht_insert(ht_t *ht, void *data)
{
	ht_record_t *record;
	hash_t bucket;
	float lf, cf;

	if(ht_lookup(ht, data) != NULL) {
		log_debug("ht_insert: duplicate entry");
		return -1;
	}

	bucket = ht->ht_hash(data) % ht->ht_buckets;

	if((record = malloc(sizeof (ht_record_t))) == NULL) {
		log_warning("ht_insert: malloc");
		return -1;
	}

	record->htr_data = data;
	record->htr_next =
		ht->ht_table[bucket] != NULL ? ht->ht_table[bucket] : NULL;

	ht->ht_table[bucket] = record;

	if(record->htr_next != NULL) {
		++ht->ht_collisions;
		
		/*
		 * First record in collision chain needs to be counted too.
		 */
		if (record->htr_next->htr_next == NULL)
		{
			++ht->ht_collisions;
		}
	}
	++ht->ht_records;

	if(bucket < ht->ht_head || ht->ht_head == 0) {
		ht->ht_head = bucket;
	}

	lf = HT_LOADFACTOR(ht);
	if (lf > 70)
	{
		log_error("ht_insert: load %.1f%", lf);
	}

	cf = HT_COLLFACTOR(ht);
	if (cf > 20)
	{
		log_error("ht_insert: collisions %.1f%", cf);
	}

	return 0;
}


void
ht_remove(ht_t *ht, void *data)
{
	ht_record_t *record;
	ht_record_t **prev;
	hash_t bucket;
	int collision = 0;

	bucket = ht->ht_hash(data) % ht->ht_buckets;

	if(ht->ht_table[bucket] == NULL) {
		log_debug("ht_remove: record not found");
		return;
	}
	
{'fraction_non_alphanumeric': 0.17353713138798826, 'fraction_numerical': 0.006484483557202408, 'mean_word_length': 3.7914201183431953, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 125, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4450096', 'n_tokens_mistral': 3126, 'n_tokens_neox': 2874, 'n_words': 549}