<commit_before>#ifndef ROLLING_HASH_H
#define ROLLING_HASH_H

#include <string>
#include <stdint.h>

// cpOff is the offset for the complement base in the random seeds table
static const int cpOff = -20;

// Table of random seeds for rolling hash function created by seedgen.cpp
static const uint64_t seedTab[256] = {
    0, 0, 0, 0, 0, 0, 0, 0, // 0..7
    0, 0, 0, 0, 0, 0, 0, 0, // 8..15
    0, 0, 0, 0, 0, 0, 0, 0, // 16..23
    0, 0, 0, 0, 0, 0, 0, 0, // 24..31
    0, 0, 0, 0, 0, 0, 0, 0, // 32..39
    0, 0, 0, 0, 0, 0x9be9b3d202074c81, 0, 0x90b45d39fb6da1fa, // 40..47
    0, 0, 0, 0x664ecc8f7cd05f14, 0, 0, 0, 0, // 48..55
    0, 0, 0, 0, 0, 0, 0, 0, // 56..63
    0x6d13226485bab26f, 0x6d13226485bab26f, 0, 0x664ecc8f7cd05f14, 0, 0, 0, 0x90b45d39fb6da1fa, // 64..71
    0, 0, 0, 0, 0, 0, 0, 0, // 72..79
    0, 0, 0, 0, 0x9be9b3d202074c81, 0, 0, 0, // 80..87
    0, 0, 0, 0, 0, 0, 0, 0, // 88..95
    0, 0, 0, 0, 0, 0, 0, 0, // 96..103
    0, 0, 0, 0, 0, 0, 0, 0, // 104..111
    0, 0, 0, 0, 0, 0, 0, 0, // 112..119
    0, 0, 0, 0, 0, 0, 0, 0, // 120..127
    0, 0, 0, 0, 0, 0, 0, 0, // 128..135
    0, 0, 0, 0, 0, 0, 0, 0, // 136..143
    0, 0, 0, 0, 0, 0, 0, 0, // 144..151
    0, 0, 0, 0, 0, 0, 0, 0, // 152..159
    0, 0, 0, 0, 0, 0, 0, 0, // 160..167
    0, 0, 0, 0, 0, 0, 0, 0, // 168..175
    0, 0, 0, 0, 0, 0, 0, 0, // 176..183
    0, 0, 0, 0, 0, 0, 0, 0, // 184..191
    0, 0, 0, 0, 0, 0, 0, 0, // 192..199
    0, 0, 0, 0, 0, 0, 0, 0, // 200..207
    0, 0, 0, 0, 0, 0, 0, 0, // 208..215
    0, 0, 0, 0, 0, 0, 0, 0, // 216..223
    0, 0, 0, 0, 0, 0, 0, 0, // 224..231
    0, 0, 0, 0, 0, 0, 0, 0, // 232..239
    0, 0, 0, 0, 0, 0, 0, 0, // 240..247
    0, 0, 0, 0, 0, 0, 0, 0  // 248..255
};

// Rotate "value" to the left by "shift" positions
static inline uint64_t rol(uint64_t value, int shift) {
    return (value << shift) | (value >> (64 - shift));
}

// Rotate "value" to the right by "shift" positions
static inline uint64_t ror(uint64_t value, int shift) {
    return (value >> shift) | (value << (64 - shift));
}

// Get forward-strand hash value of the base kmer, i.e. fhval(kmer_0)
static inline uint64_t getFhval(const std::string &kmerSeq) {
    uint64_t hVal=0;
    unsigned k = kmerSeq.length();
    for(unsigned i=0; i<k; i++)
        hVal ^= rol(seedTab[(unsigned)kmerSeq[i]], k-1-i);
    return hVal;
}

// Get reverse-strand hash value of the base kmer, i.e. rhval(kmer_0)
static inline uint64_t getRhval(const std::string &kmerSeq) {
    uint64_t hVal=0;
    unsigned k = kmerSeq.length();
    for(unsigned i=0; i<k; i++)
        hVal ^= rol(seedTab[(unsigned)kmerSeq[i]+cpOff], i);
    return hVal;
}
<commit_msg>rolling.h: seedTab updated with better random seeds<commit_after>#ifndef ROLLING_HASH_H
#define ROLLING_HASH_H

#include <string>
#include <stdint.h>

// cpOff is the offset for the complement base in the random seeds table
static const int cpOff = -20;

// Table of random seeds for rolling hash function created by seedgen.cpp
/*static const uint64_t seedTab[256] = {
    0, 0, 0, 0, 0, 0, 0, 0, // 0..7
    0, 0, 0, 0, 0, 0, 0, 0, // 8..15
    0, 0, 0, 0, 0, 0, 0, 0, // 16..23
    0, 0, 0, 0, 0, 0, 0, 0, // 24..31
    0, 0, 0, 0, 0, 0, 0, 0, // 32..39
    0, 0, 0, 0, 0, 0x9be9b3d202074c81, 0, 0x90b45d39fb6da1fa, // 40..47
    0, 0, 0, 0x664ecc8f7cd05f14, 0, 0, 0, 0, // 48..55
    0, 0, 0, 0, 0, 0, 0, 0, // 56..63
    0x6d13226485bab26f, 0x6d13226485bab26f, 0, 0x664ecc8f7cd05f14, 0, 0, 0, 0x90b45d39fb6da1fa, // 64..71
    0, 0, 0, 0, 0, 0, 0, 0, // 72..79
    0, 0, 0, 0, 0x9be9b3d202074c81, 0, 0, 0, // 80..87
    0, 0, 0, 0, 0, 0, 0, 0, // 88..95
    0, 0, 0, 0, 0, 0, 0, 0, // 96..103
    0, 0, 0, 0, 0, 0, 0, 0, // 104..111
    0, 0, 0, 0, 0, 0, 0, 0, // 112..119
    0, 0, 0, 0, 0, 0, 0, 0, // 120..127
    0, 0, 0, 0, 0, 0, 0, 0, // 128..135
    0, 0, 0, 0, 0, 0, 0, 0, // 136..143
    0, 0, 0, 0, 0, 0, 0, 0, // 144..151
    0, 0, 0, 0, 0, 0, 0, 0, // 152..159
    0, 0, 0, 0, 0, 0, 0, 0, // 160..167
    0, 0, 0, 0, 0, 0, 0, 0, // 168..175
    0, 0, 0, 0, 0, 0, 0, 0, // 176..183
    0, 0, 0, 0, 0, 0, 0, 0, // 184..191
    0, 0, 0, 0, 0, 0, 0, 0, // 192..199
    0, 0, 0, 0, 0, 0, 0, 0, // 200..207
    0, 0, 0, 0, 0, 0, 0, 0, // 208..215
    0, 0, 0, 0, 0, 0, 0, 0, // 216..223
    0, 0, 0, 0, 0, 0, 0, 0, // 224..231
    0, 0, 0, 0, 0, 0, 0, 0, // 232..239
    0, 0, 0, 0, 0, 0, 0, 0, // 240..247
    0, 0, 0, 0, 0, 0, 0, 0  // 248..255
};*/
static const uint64_t seedTab[256] = {
    0, 0, 0, 0, 0, 0, 0, 0, // 0..7
    0, 0, 0, 0, 0, 0, 0, 0, // 8..15
    0, 0, 0, 0, 0, 0, 0, 0, // 16..23
    0, 0, 0, 0, 0, 0, 0, 0, // 24..31
    0, 0, 0, 0, 0, 0, 0, 0, // 32..39
    0, 0, 0, 0, 0, 0xd795b53f4f2d1046, 0, 0x5c600a20e280215e, // 40..47
    0, 0, 0, 0xab0b70d8997adea1, 0, 0, 0, 0, // 48..55
    0, 0, 0, 0, 0, 0, 0, 0, // 56..63
    0x20fecfc734d7efb9, 0x20fecfc734d7efb9, 0, 0xab0b70d8997adea1, 0, 0, 0, 0x5c600a20e280215e, // 64..71
    0, 0, 0, 0, 0, 0, 0, 0, // 72..79
    0, 0, 0, 0, 0xd795b53f4f2d1046, 0, 0, 0, // 80..87
    0, 0, 0, 0, 0, 0, 0, 0, // 88..95
    0, 0, 0, 0, 0, 0, 0, 0, // 96..103
    0, 0, 0, 0, 0, 0, 0, 0, // 104..111
    0, 0, 0, 0, 0, 0, 0, 0, // 112..119
    0, 0, 0, 0, 0, 0, 0, 0, // 120..127
    0, 0, 0, 0, 0, 0, 0, 0, // 128..135
    0, 0, 0, 0, 0, 0, 0, 0, // 136..143
    0, 0, 0, 0, 0, 0, 0, 0, // 144..151
    0, 0, 0, 0, 0, 0, 0, 0, // 152..159
    0, 0, 0, 0, 0, 0, 0, 0, // 160..167
    0, 0, 0, 0, 0, 0, 0, 0, // 168..175
    0, 0, 0, 0, 0, 0, 0, 0, // 176..183
    0, 0, 0, 0, 0, 0, 0, 0, // 184..191
    0, 0, 0, 0, 0, 0, 0, 0, // 192..199
    0, 0, 0, 0, 0, 0, 0, 0, // 200..207
    0, 0, 0, 0, 0, 0, 0, 0, // 208..215
    0, 0, 0, 0, 0, 0, 0, 0, // 216..223
    0, 0, 0, 0, 0, 0, 0, 0, // 224..231
    0, 0, 0, 0, 0, 0, 0, 0, // 232..239
    0, 0, 0, 0, 0, 0, 0, 0, // 240..247
    0, 0, 0, 0, 0, 0, 0, 0  // 248..255
};


// Rotate "value" to the left by "shift" positions
static inline uint64_t rol(uint64_t value, int shift) {
    return (value << shift) | (value >> (64 - shift));
}

// Rotate "value" to the right by "shift" positions
static inline uint64_t ror(uint64_t value, int shift) {
    return (value >> shift) | (value << (64 - shift));
}

// Get forward-strand hash value of the base kmer, i.e. fhval(kmer_0)
static inline uint64_t getFhval(const std::string &kmerSeq) {
    uint64_t hVal=0;
    unsigned k = kmerSeq.length();
    for(unsigned i=0; i<k; i++)
        hVal ^= rol(seedTab[(unsigned)kmerSeq[i]], k-1-i);
    return hVal;
}

// Get reverse-strand hash value of the base kmer, i.e. rhval(kmer_0)
static inline uint64_t getRhval(const std::string &kmerSeq) {
    uint64_t hVal=0;
    unsigned k = kmerSeq.length();
    for(unsigned i=0; i<k; i++)
        hVal ^= rol(seedTab[(unsigned)kmerSeq[i]+cpOff], i);
    return hVal;
}

{'fraction_non_alphanumeric': 0.21941862180906005, 'fraction_numerical': 0.23092813929467315, 'mean_word_length': 1.9793406593406593, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4344589', 'n_tokens_mistral': 4527, 'n_tokens_neox': 3316, 'n_words': 1160}