<commit_before>#include <stdio.h>
#include <stdint.h>

#include "m-array.h"
#include "m-dict.h"
#include "m-string.h"
#include "m-algo.h"

/* Based on:
 *  Easy Perfect Minimal Hashing 
 *  By <NAME>. Released to the public domain.
 *  http://stevehanov.ca/blog/index.php?id=119
 *
 * Based on:
 * <NAME>, <NAME>, <NAME> and <NAME>, 
 * "Practical minimal perfect hash functions for large databases", CACM, 35(1):105-121
 *
 * also a good reference:
 *  Compress, Hash, and Displace algorithm by <NAME>,
 * <NAME>, and <NAME>
 */



/* Define the needed data structure : */

// Define a dictionnary of constant string --> integer
DICT_DEF2(dict_mph, const char *, M_CSTR_OPLIST, uint32_t, M_DEFAULT_OPLIST)
#define M_OPL_dict_mph_t() DICT_OPLIST(dict_mph, M_CSTR_OPLIST, M_DEFAULT_OPLIST)

// Defin an array of integer
ARRAY_DEF(array_value, uint32_t) // Start from 1. 0 is 'None'
#define M_OPL_array_value_t() ARRAY_OPLIST(array_value)

// Defin an array of integer
ARRAY_DEF(array_seed, int32_t)
#define M_OPL_array_seed_t() ARRAY_OPLIST(array_seed)
ALGO_DEF(array_seed, M_OPL_array_seed_t())

// Define an array of constant string that are ordered by their size.
ARRAY_DEF(array_cstr, const char *, M_CSTR_OPLIST)
static inline int array_cstr_cmp(const array_cstr_t a, const array_cstr_t b) {
  size_t sa = array_cstr_size(a);
  size_t sb = array_cstr_size(b);
  return (sa > sb) ? -1 : (sa < sb);
}
#define M_OPL_array_cstr_t() M_OPEXTEND(ARRAY_OPLIST(array_cstr, M_CSTR_OPLIST), CMP(array_cstr_cmp))

// Define an array of array of constant string, and some algo over this container
ARRAY_DEF(array_bucket, array_cstr_t)
#define M_OPL_array_bucket_t() ARRAY_OPLIST(array_bucket, M_OPL_array_cstr_t() )
ALGO_DEF(array_bucket, M_OPL_array_bucket_t())

// Define an array of string to save the input.
ARRAY_DEF(array_string, string_t)
#define M_OPL_array_string_t() ARRAY_OPLIST(array_string, M_OPL_string_t())




// The algorithm may not converge on all inputs.
// Example: dictionnary with only 'a' and 'e'
// The hash function doesn't provide the needed property.
static uint32_t hash(int32_t seed, const char str[])
{
  if (seed == 0) seed = 0x01000193;
  while (*str) {
    seed = ((seed * 0x01000193) ^ *str);
    str++;
  }
  return seed;
}

<commit_msg>Update comment.<commit_after>#include <stdio.h>
#include <stdint.h>

#include "m-array.h"
#include "m-dict.h"
#include "m-string.h"
#include "m-algo.h"

/* Based on:
 *  Easy Perfect Minimal Hashing 
 *  By <NAME>. Released to the public domain.
 *  http://stevehanov.ca/blog/index.php?id=119
 *
 * Based on:
 * <NAME>, <NAME>, <NAME> and <NAME>, 
 * "Practical minimal perfect hash functions for large databases", CACM, 35(1):105-121
 *
 * also a good reference:
 *  Compress, Hash, and Displace algorithm by <NAME>,
 * <NAME>, and <NAME>
 */



/* Define the needed data structure : */

// Define a dictionnary of constant string --> integer
DICT_DEF2(dict_mph, const char *, M_CSTR_OPLIST, uint32_t, M_DEFAULT_OPLIST)
#define M_OPL_dict_mph_t() DICT_OPLIST(dict_mph, M_CSTR_OPLIST, M_DEFAULT_OPLIST)

// Defin an array of integer
ARRAY_DEF(array_value, uint32_t) // Start from 1. 0 is 'None'
#define M_OPL_array_value_t() ARRAY_OPLIST(array_value)

// Defin an array of integer
ARRAY_DEF(array_seed, int32_t)
#define M_OPL_array_seed_t() ARRAY_OPLIST(array_seed)
ALGO_DEF(array_seed, M_OPL_array_seed_t())

// Define an array of constant string that are ordered by their size.
ARRAY_DEF(array_cstr, const char *, M_CSTR_OPLIST)
static inline int array_cstr_cmp(const array_cstr_t a, const array_cstr_t b) {
  size_t sa = array_cstr_size(a);
  size_t sb = array_cstr_size(b);
  return (sa > sb) ? -1 : (sa < sb);
}
#define M_OPL_array_cstr_t() M_OPEXTEND(ARRAY_OPLIST(array_cstr, M_CSTR_OPLIST), CMP(array_cstr_cmp))

// Define an array of array of constant string, and some algo over this container
ARRAY_DEF(array_bucket, array_cstr_t)
#define M_OPL_array_bucket_t() ARRAY_OPLIST(array_bucket, M_OPL_array_cstr_t() )
ALGO_DEF(array_bucket, M_OPL_array_bucket_t())

// Define an array of string to save the input.
ARRAY_DEF(array_string, string_t)
#define M_OPL_array_string_t() ARRAY_OPLIST(array_string, M_OPL_string_t())




// The algorithm may not converge on all inputs.
// Example: dictionnary with only 'a' and 'c' for which there is no seed
// where the hash value diverge on the final bit.
// The hash function doesn't provide the needed property.
static uint32_t hash(int32_t seed, const char str[])
{
  if (seed == 0) seed = 0x01000193;
  while (*str) {
    seed = ((seed * 0x01000193) ^ *str);
    str++;
  }
  return seed;
}


{'fraction_non_alphanumeric': 0.11591057087041459, 'fraction_numerical': 0.019535489472541783, 'mean_word_length': 4.818181818181818, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '571900', 'n_tokens_mistral': 1819, 'n_tokens_neox': 1727, 'n_words': 478}