<commit_before>#!/usr/bin/env lua

-- Defaults
operation_mode='p'
verbosity='v'
freqfile=nil
infile=nil
outfile=nil
rulefile="anglofrench"
instream=io.stdin
outstream=io.stdout

-- Hashing parameters
-- TODO: seed the RNG
-- TODO: I hereby promise to study the theory 
-- and improve hashing function after 1st april
hash_p=524287 -- Mersenne prime 2^19-1
hash_a=math.random(0, hash_p-1)

-- Output code templates
codetempl_header=[[
/* THIS FILE IS GENERATED AUTOMATICALLY BY AZARIFY AND NOT INTENDED FOR MANUAL MODIFICATION
 * As a matter of fact, don't read it either, it will haunt you at night.
 * You have been warned.
 */
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdio.h>

]]

codetemp_footer=[[

static uint64_t azarify_hash_iteration(uint64_t h, char c)
{
    /* TODO: Mersenne prime fast modulo */
    return (h*azarify_hash_a + c) % azarify_hash_p;
}

static const char* azarify_hash_lookup_value(const char *needle, uint64_t hash, size_t start, size_t end, short ending)
{
    /* TODO: Maybe select the buckets size from Mersenne primes too? */
    unsigned i=hash % azarify_hash_buckets;
    <commit_msg>Removed unneeded header<commit_after>#!/usr/bin/env lua

-- Defaults
operation_mode='p'
verbosity='v'
freqfile=nil
infile=nil
outfile=nil
rulefile="anglofrench"
instream=io.stdin
outstream=io.stdout

-- Hashing parameters
-- TODO: seed the RNG
-- TODO: I hereby promise to study the theory 
-- and improve hashing function after 1st april
hash_p=524287 -- Mersenne prime 2^19-1
hash_a=math.random(0, hash_p-1)

-- Output code templates
codetempl_header=[[
/* THIS FILE IS GENERATED AUTOMATICALLY BY AZARIFY AND NOT INTENDED FOR MANUAL MODIFICATION
 * As a matter of fact, don't read it either, it will haunt you at night.
 * You have been warned.
 */
#include <string.h>
#include <stdlib.h>
#include <stdint.h>

]]

codetemp_footer=[[

static uint64_t azarify_hash_iteration(uint64_t h, char c)
{
    /* TODO: Mersenne prime fast modulo */
    return (h*azarify_hash_a + c) % azarify_hash_p;
}

static const char* azarify_hash_lookup_value(const char *needle, uint64_t hash, size_t start, size_t end, short ending)
{
    /* TODO: Maybe select the buckets size from Mersenne primes too? */
    unsigned i=hash % azarify_hash_buckets;
    
{'fraction_non_alphanumeric': 0.10162067455102934, 'fraction_numerical': 0.016644765659220323, 'mean_word_length': 4.386792452830188, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3246695', 'n_tokens_mistral': 855, 'n_tokens_neox': 806, 'n_words': 250}