<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "runtime.h"
#include "hashmap.h"

/* Return a pointer to the struct/union of type "type"
   whose "field" field is addressed by pointer "p". */

struct hash {	   /* a hash table; initialize with hash_init() */
	uint32 count;	  /* elements in table - must be first */

	uint8 datasize;   /* amount of data to store in entry */
	uint8 max_power;  /* max power of 2 to create sub-tables */
	uint8 max_probes; /* max entries to probe before rehashing */
	int32 changes;	      /* inc'ed whenever a subtable is created/grown */
	hash_hash_t (*data_hash) (uint32, void *a);  /* return hash of *a */
	uint32 (*data_eq) (uint32, void *a, void *b);   /* return whether *a == *b */
	void (*data_del) (uint32, void *arg, void *data);  /* invoked on deletion */
	struct hash_subtable *st;    /* first-level table */

	uint32	keysize;
	uint32	valsize;
	uint32	ko;
	uint32	vo;
	uint32	po;
	Alg*	keyalg;
	Alg*	valalg;
};

struct hash_entry {
	hash_hash_t hash;     /* hash value of data */
	byte data[1];	 /* user data has "datasize" bytes */
};

struct hash_subtable {
	uint8 power;	 /* bits used to index this table */
	uint8 used;	  /* bits in hash used before reaching this table */
	uint8 datasize;      /* bytes of client data in an entry */
	uint8 max_probes;    /* max number of probes when searching */
	int16 limit_bytes;	   /* max_probes * (datasize+sizeof (hash_hash_t)) */
	struct hash_entry *end;      /* points just past end of entry[] */
	struct hash_entry entry[1];  /* 2**power+max_probes-1 elements of elemsize bytes */
};

#define HASH_DATA_EQ(h,x,y) ((*h->data_eq) (h->keysize, (x), (y)))

#define HASH_REHASH 0x2       /* an internal flag */
/* the number of bits used is stored in the flags word too */
#define HASH_USED(x)      ((x) >> 2)
#define HASH_MAKE_USED(x) ((x) << 2)

#define HASH_LOW	6
#define HASH_ONE	(((hash_hash_t)1) << HASH_LOW)
#define HASH_MASK       (HASH_ONE - 1)
#define HASH_ADJUST(x)  (((x) < HASH_ONE) << HASH_LOW)

#define HASH_BITS       (sizeof (hash_hash_t) * 8)

#define HASH_SUBHASH    HASH_MASK
#define HASH_NIL	0
#define HASH_NIL_MEMSET 0

#define HASH_OFFSET(base, byte_offset) \
	  ((struct hash_entry *) (((byte *) (base)) + (byte_offset)))


/* return a hash layer with 2**power empty entries */
static struct hash_subtable *
hash_subtable_new (struct hash *h, int32 power, int32 used)
{
	int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]);
	int32 bytes = elemsize << power;
	struct hash_subtable *st;
	int32 limit_bytes = h->max_probes * elemsize;
	int32 max_probes = h->max_probes;

	if (bytes < limit_bytes) {
		limit_bytes = bytes;
		max_probes = 1 << power;
	}
	bytes += limit_bytes - elemsize;
	st = malloc (offsetof (struct hash_subtable, entry[0]) + bytes);
	st->power = power;
	st->used = used;
	st->datasize = h->datasize;
	st->max_probes = max_probes;
	st->limit_bytes = limit_bytes;
	st->end = HASH_OFFSET (st->entry, bytes);
	memset (st->entry, HASH_NIL_MEMSET, bytes);
	return (st);
}

static void
init_sizes (int64 hint, int32 *init_power, int32 *max_power)
{
	int32 log = 0;
	int32 i;

	for (i = 32; i != 0; i >>= 1) {
		if ((hint >> (log + i)) != 0) {
			log += i;
		}
	}
	log += 1 + (((hint << 3) >> log) >= 11);  /* round up for utilization */
	if (log <= 14) {
		*init_power = log;
	} else {
		*init_power = 12;
	}
	*max_power = 12;
}

static void
hash_init (struct hash *h,
		int32 datasize,
		hash_hash_t (*data_hash) (uint32, void *),
		uint32 (*data_eq) (uint32, void *, void *),
		void (*data_del) (uint32, void *, void *),
		int64 hint)
{
	int32 init_power;
	int32 max_power;

	if(datasize < sizeof (void *))
		datasize = sizeof (void *);
	init_sizes (hint, &init_power, &max_power);
	h->datasize = datasize;
	h->max_power = max_power;
	h->max_probes = 15;
	assert (h->datasize == datasize);
	assert (h->max_power == max_power);
	assert (sizeof (void *) <= h->datasize || h->max_power == 255);
	h->count = 0;
	h->changes = 0;
	h->data_hash = data_hash;
	h->data_eq = data_eq;
	h->data_del = data_del;
	h->st = hash_subtable_new (h, init_power, 0);
}

static void
hash_remove_n (struct hash_subtable *st, struct hash_entry *dst_e, int32 n)
{<commit_msg>temp fix for map not multiple of 8<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "runtime.h"
#include "hashmap.h"

/* Return a pointer to the struct/union of type "type"
   whose "field" field is addressed by pointer "p". */

struct hash {	   /* a hash table; initialize with hash_init() */
	uint32 count;	  /* elements in table - must be first */

	uint8 datasize;   /* amount of data to store in entry */
	uint8 max_power;  /* max power of 2 to create sub-tables */
	uint8 max_probes; /* max entries to probe before rehashing */
	int32 changes;	      /* inc'ed whenever a subtable is created/grown */
	hash_hash_t (*data_hash) (uint32, void *a);  /* return hash of *a */
	uint32 (*data_eq) (uint32, void *a, void *b);   /* return whether *a == *b */
	void (*data_del) (uint32, void *arg, void *data);  /* invoked on deletion */
	struct hash_subtable *st;    /* first-level table */

	uint32	keysize;
	uint32	valsize;
	uint32	ko;
	uint32	vo;
	uint32	po;
	Alg*	keyalg;
	Alg*	valalg;
};

struct hash_entry {
	hash_hash_t hash;     /* hash value of data */
	byte data[1];	 /* user data has "datasize" bytes */
};

struct hash_subtable {
	uint8 power;	 /* bits used to index this table */
	uint8 used;	  /* bits in hash used before reaching this table */
	uint8 datasize;      /* bytes of client data in an entry */
	uint8 max_probes;    /* max number of probes when searching */
	int16 limit_bytes;	   /* max_probes * (datasize+sizeof (hash_hash_t)) */
	struct hash_entry *end;      /* points just past end of entry[] */
	struct hash_entry entry[1];  /* 2**power+max_probes-1 elements of elemsize bytes */
};

#define HASH_DATA_EQ(h,x,y) ((*h->data_eq) (h->keysize, (x), (y)))

#define HASH_REHASH 0x2       /* an internal flag */
/* the number of bits used is stored in the flags word too */
#define HASH_USED(x)      ((x) >> 2)
#define HASH_MAKE_USED(x) ((x) << 2)

#define HASH_LOW	6
#define HASH_ONE	(((hash_hash_t)1) << HASH_LOW)
#define HASH_MASK       (HASH_ONE - 1)
#define HASH_ADJUST(x)  (((x) < HASH_ONE) << HASH_LOW)

#define HASH_BITS       (sizeof (hash_hash_t) * 8)

#define HASH_SUBHASH    HASH_MASK
#define HASH_NIL	0
#define HASH_NIL_MEMSET 0

#define HASH_OFFSET(base, byte_offset) \
	  ((struct hash_entry *) (((byte *) (base)) + (byte_offset)))


/* return a hash layer with 2**power empty entries */
static struct hash_subtable *
hash_subtable_new (struct hash *h, int32 power, int32 used)
{
	int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]);
	int32 bytes = elemsize << power;
	struct hash_subtable *st;
	int32 limit_bytes = h->max_probes * elemsize;
	int32 max_probes = h->max_probes;

	if (bytes < limit_bytes) {
		limit_bytes = bytes;
		max_probes = 1 << power;
	}
	bytes += limit_bytes - elemsize;
	st = malloc (offsetof (struct hash_subtable, entry[0]) + bytes);
	st->power = power;
	st->used = used;
	st->datasize = h->datasize;
	st->max_probes = max_probes;
	st->limit_bytes = limit_bytes;
	st->end = HASH_OFFSET (st->entry, bytes);
	memset (st->entry, HASH_NIL_MEMSET, bytes);
	return (st);
}

static void
init_sizes (int64 hint, int32 *init_power, int32 *max_power)
{
	int32 log = 0;
	int32 i;

	for (i = 32; i != 0; i >>= 1) {
		if ((hint >> (log + i)) != 0) {
			log += i;
		}
	}
	log += 1 + (((hint << 3) >> log) >= 11);  /* round up for utilization */
	if (log <= 14) {
		*init_power = log;
	} else {
		*init_power = 12;
	}
	*max_power = 12;
}

static void
hash_init (struct hash *h,
		int32 datasize,
		hash_hash_t (*data_hash) (uint32, void *),
		uint32 (*data_eq) (uint32, void *, void *),
		void (*data_del) (uint32, void *, void *),
		int64 hint)
{
	int32 init_power;
	int32 max_power;

	if(datasize < sizeof (void *))
		datasize = sizeof (void *);
	datasize = rnd(datasize, 8);
	init_sizes (hint, &init_power, &max_power);
	h->datasize = datasize;
	h->max_power = max_power;
	h->max_probes = 15;
	assert (h->datasize == datasize);
	assert (h->max_power == max_power);
	assert (sizeof (void *) <= h->datasize || h->max_power == 255);
	h->count = 0;
	h->changes = 0;
	h->data_hash = data_hash;
	h->data_eq = data_eq;
	h->data_del = data_del;
	h->st = hash_subtable_new (h, init_power, 0);
}

static void
hash_remove_n (struct hash_subtable *st, struct hash_entry *dst_e, int32 n)
{
{'fraction_non_alphanumeric': 0.1436127400252961, 'fraction_numerical': 0.027365758307462344, 'mean_word_length': 3.5899736147757255, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1740263', 'n_tokens_mistral': 3493, 'n_tokens_neox': 3262, 'n_words': 1010}