<commit_before>#ifndef SHORTHASH_TABULATED_H
#define SHORTHASH_TABULATED_H

#include <stdint.h>
#include <limits.h>
#include "util.h"


/**
Cyclic is analyzed in
"Recursive n-gram hashing is pairwise independent, at best"
*/

typedef struct cyclic_s {
    uint64_t hashtab[1 << CHAR_BIT];
} cyclic_t;

void cyclic_init(cyclic_t *k) {
        for (uint32_t j = 0; j < (1 << CHAR_BIT); j++) {
            k->hashtab[j] = get64rand();
        }
}

// shoudl get compiled to ror on x64
static inline uint64_t rotate(uint64_t x, int r) {
  return (x >> r) | (x << (64 - r));
}

__attribute__((always_inline))
inline uint64_t cyclic(uint64_t val, const cyclic_t *k) {
    uint64_t h = 0;
    const unsigned char *s = (const unsigned char *)&val;
    h ^= k->hashtab[s[0]];
    h ^= rotate(k->hashtab[s[1]],1);
    h ^= rotate(k->hashtab[s[2]],2);
    h ^= rotate(k->hashtab[s[3]],3);
    h ^= rotate(k->hashtab[s[4]],4);
    h ^= rotate(k->hashtab[s[5]],5);
    h ^= rotate(k->hashtab[s[6]],6);
    h ^= rotate(k->hashtab[s[7]],7);
    return h;
}

typedef struct cyclic32_s {
    uint32_t hashtab[1 << CHAR_BIT];
} cyclic32_t;

void cyclic32_init(cyclic32_t *k) {
        for (uint32_t j = 0; j < (1 << CHAR_BIT); j++) {<commit_msg>Fixing typo<commit_after>#ifndef SHORTHASH_TABULATED_H
#define SHORTHASH_TABULATED_H

#include <stdint.h>
#include <limits.h>
#include "util.h"


/**
Cyclic is analyzed in
"Recursive n-gram hashing is pairwise independent, at best"
*/

typedef struct cyclic_s {
    uint64_t hashtab[1 << CHAR_BIT];
} cyclic_t;

void cyclic_init(cyclic_t *k) {
        for (uint32_t j = 0; j < (1 << CHAR_BIT); j++) {
            k->hashtab[j] = get64rand();
        }
}

// should get compiled to ror on x64
static inline uint64_t rotate(uint64_t x, int r) {
  return (x >> r) | (x << (64 - r));
}

__attribute__((always_inline))
inline uint64_t cyclic(uint64_t val, const cyclic_t *k) {
    uint64_t h = 0;
    const unsigned char *s = (const unsigned char *)&val;
    h ^= k->hashtab[s[0]];
    h ^= rotate(k->hashtab[s[1]],1);
    h ^= rotate(k->hashtab[s[2]],2);
    h ^= rotate(k->hashtab[s[3]],3);
    h ^= rotate(k->hashtab[s[4]],4);
    h ^= rotate(k->hashtab[s[5]],5);
    h ^= rotate(k->hashtab[s[6]],6);
    h ^= rotate(k->hashtab[s[7]],7);
    return h;
}

typedef struct cyclic32_s {
    uint32_t hashtab[1 << CHAR_BIT];
} cyclic32_t;

void cyclic32_init(cyclic32_t *k) {
        for (uint32_t j = 0; j < (1 << CHAR_BIT); j++) {
{'fraction_non_alphanumeric': 0.17862969004893964, 'fraction_numerical': 0.04404567699836868, 'mean_word_length': 3.4762773722627736, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '614627', 'n_tokens_mistral': 1126, 'n_tokens_neox': 1021, 'n_words': 235}