<commit_before>#ifndef GLEAN_H
#define GLEAN_H

#define GLN_VERSION_STRING "000100" /* 0.1.0 "beta vulgaris" */

#ifdef NDEBUG
#define DEBUG 0
#else
#define DEBUG 0
#endif

#define DB_DEBUG (DEBUG || 0)

#define DEBUG_HASH 0

#define MAX_WORD_SZ 512
#define MIN_WORD_SZ 3

#define DEF_ZLIB_COMPRESS 6
#define PROFILE_COMPRESSION 0

/* Compression helps quite a bit more with the filename DB than the token DB,
 * but it's cheap, so use it. */
#define SKIP_COMPRESS 0

/* Should we use u_int16_t or u_int32_t for hashes? */
#define FOUR_BYTE_HASH 0

#if FOUR_BYTE_HASH
typedef u_int32_t hash_t;
#else
typedef u_int16_t hash_t;
#endif

#define HASH_BYTES sizeof(hash_t)

/* How many tokenizer processes? */
#define DEF_WORKER_CT 8
#define MAX_WORKER_CT 64

/* Number of 'X's to use for database data alignment */
#define DB_X_CT 1

/* If using stop words, how many relatively-flat differences are
 * needed before stopping. TODO configure */
#define FLAT_CT 5
#define CHANGE_FACTOR 0.75

/* If file's first read has less % printable bytes than this, skip it. */
#define MIN_TOKEN_PRINTABLE 0.8

/* How many matches per token is worth warning about? */
#define TOO_MANY_MATCHES 25

/* When showing progress, print a message every time this many files are enqueued */
#define ENQUEUE_PROGRESS_CT 1000

/* Just die if we ever use more than this. */<commit_msg>Switch back to 4-byte hashes, for now.<commit_after>#ifndef GLEAN_H
#define GLEAN_H

#define GLN_VERSION_STRING "000100" /* 0.1.0 "beta vulgaris" */

#ifdef NDEBUG
#define DEBUG 0
#else
#define DEBUG 0
#endif

#define DB_DEBUG (DEBUG || 0)

#define DEBUG_HASH 0

#define MAX_WORD_SZ 512
#define MIN_WORD_SZ 3

#define DEF_ZLIB_COMPRESS 6
#define PROFILE_COMPRESSION 0

/* Compression helps quite a bit more with the filename DB than the token DB,
 * but it's cheap, so use it. */
#define SKIP_COMPRESS 0

/* Should we use u_int16_t or u_int32_t for hashes? */
#define FOUR_BYTE_HASH 1

#if FOUR_BYTE_HASH
typedef u_int32_t hash_t;
#else
typedef u_int16_t hash_t;
#endif

#define HASH_BYTES sizeof(hash_t)

/* How many tokenizer processes? */
#define DEF_WORKER_CT 8
#define MAX_WORKER_CT 64

/* Number of 'X's to use for database data alignment */
#define DB_X_CT 1

/* If using stop words, how many relatively-flat differences are
 * needed before stopping. TODO configure */
#define FLAT_CT 5
#define CHANGE_FACTOR 0.75

/* If file's first read has less % printable bytes than this, skip it. */
#define MIN_TOKEN_PRINTABLE 0.8

/* How many matches per token is worth warning about? */
#define TOO_MANY_MATCHES 25

/* When showing progress, print a message every time this many files are enqueued */
#define ENQUEUE_PROGRESS_CT 1000

/* Just die if we ever use more than this. */
{'fraction_non_alphanumeric': 0.07934186471663619, 'fraction_numerical': 0.033272394881170016, 'mean_word_length': 4.354207436399217, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6556803', 'n_tokens_mistral': 1038, 'n_tokens_neox': 942, 'n_words': 366}