<commit_before>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* Copyright 2019 <NAME>                                         */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

#include "const-prime.h"
#include "connectors.h"
#include "tracon-set.h"
#include "utilities.h"

/**
 * This is an adaptation of the string_set module for detecting unique
 * connector trailing sequences (aka tracons).
 *
 * It is used to generate tracon encoding for the parsing and for the
 * pruning steps. Not like string_set, the actual hash values here
 * are external.
 *
 * A tracon is identified by (the address of) its first connector.
 *
 * The API here is similar to that of string_set, with these differences:
 *
 * 1. tracon_set_add() returns a pointer to the hash slot of the tracon if
 * it finds it. Else it returns a pointer to a NULL hash slot, and
 * the caller is expected to assign to it the tracon (its address
 * after it is copied to the destination buffer).
 *
 * 2. A new API tracon_set_shallow() is used to require that tracons
 * which start with a shallow connector will not be considered the same
 * as ones that start with a deep connector. The power pruning algo
 * depends on that.
 *
 * 3. A new API tracon_set_reset() is used to clear the hash table slots
 * (only, their value remains intact).
 */

static unsigned int hash_connectors(const Connector *c, unsigned int shallow)
{
	unsigned int accum = shallow && c->shallow;

	for (; c != NULL; c = c->next)
	{
		accum = (7 * accum) +
		((c->desc->uc_num)<<18) +
		(((unsigned int)c->multi)<<31) +
		(unsigned int)c->desc->lc_letters;
	}

	return accum;
}

static unsigned int find_prime_for(size_t count)
{
	size_t i;
	for (i = 0; i < MAX_S_PRIMES; i ++)
		if ((8 * count) < (3 * s_prime[i])) return i;

	assert(0, "find_prime_for(%zu): Absurdly big count", count);
	return 0;
}
<commit_msg>tracon-set.c hash_connectors(): Change multiplication constant 7 -> 19<commit_after>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* Copyright 2019 <NAME>                                         */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

#include "const-prime.h"
#include "connectors.h"
#include "tracon-set.h"
#include "utilities.h"

/**
 * This is an adaptation of the string_set module for detecting unique
 * connector trailing sequences (aka tracons).
 *
 * It is used to generate tracon encoding for the parsing and for the
 * pruning steps. Not like string_set, the actual hash values here
 * are external.
 *
 * A tracon is identified by (the address of) its first connector.
 *
 * The API here is similar to that of string_set, with these differences:
 *
 * 1. tracon_set_add() returns a pointer to the hash slot of the tracon if
 * it finds it. Else it returns a pointer to a NULL hash slot, and
 * the caller is expected to assign to it the tracon (its address
 * after it is copied to the destination buffer).
 *
 * 2. A new API tracon_set_shallow() is used to require that tracons
 * which start with a shallow connector will not be considered the same
 * as ones that start with a deep connector. The power pruning algo
 * depends on that.
 *
 * 3. A new API tracon_set_reset() is used to clear the hash table slots
 * (only, their value remains intact).
 */

static unsigned int hash_connectors(const Connector *c, unsigned int shallow)
{
	unsigned int accum = shallow && c->shallow;

	for (; c != NULL; c = c->next)
	{
		accum = (19 * accum) +
		((c->desc->uc_num)<<18) +
		(((unsigned int)c->multi)<<31) +
		(unsigned int)c->desc->lc_letters;
	}

	return accum;
}

static unsigned int find_prime_for(size_t count)
{
	size_t i;
	for (i = 0; i < MAX_S_PRIMES; i ++)
		if ((8 * count) < (3 * s_prime[i])) return i;

	assert(0, "find_prime_for(%zu): Absurdly big count", count);
	return 0;
}

{'fraction_non_alphanumeric': 0.14771417595082598, 'fraction_numerical': 0.008835958509412217, 'mean_word_length': 2.365869424692954, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1042657', 'n_tokens_mistral': 1408, 'n_tokens_neox': 1308, 'n_words': 588}