<commit_before>#include <stdio.h>
#include <unistd.h>
#include <assert.h>
#include <stdbool.h>
#include "rabin.h"

// 1MiB buffer
uint8_t buf[1024*1024];
size_t bytes;

int main(void) {
    struct rabin_t *hash;
    hash = rabin_init();

    unsigned int chunks = 0;

    while (!feof(stdin)) {
        size_t len = fread(buf, 1, sizeof(buf), stdin);
        uint8_t *ptr = &buf[0];

        // printf("read %zd bytes\n", len);

        bytes += len;

        while (1) {
            // printf("  rabin_next_chunk(%p, %zd)\n", ptr, len);
            int remaining = rabin_next_chunk(hash, ptr, len);

            // printf("  rabin remaining: %d bytes, len %zd\n", remaining, len);

            if (remaining < 0) {
                break;
            }

            len -= remaining;
            ptr += remaining;

            // printf("remaining: %d, len %zd, ptr %p\n", remaining, len, ptr);

            printf("%d %016llx\n",
                last_chunk.length,
                (long long unsigned int)last_chunk.cut_fingerprint);

            chunks++;

            // printf("  next bytes: %02x%02x%02x%02x\n", *ptr, *(ptr+1), *(ptr+2), *(ptr+3));
        }
    }

    rabin_finalize(hash);
    chunks++;

    printf("%d %016llx\n",
        last_chunk.length,
        (long long unsigned int)last_chunk.cut_fingerprint);

    unsigned int avg = 0;
    if (chunks > 0)
        avg = bytes / chunks;
    printf("%d chunks, average chunk size %d\n", chunks, avg);

    return 0;<commit_msg>Remove debug printf()<commit_after>#include <stdio.h>
#include <unistd.h>
#include <assert.h>
#include <stdbool.h>
#include "rabin.h"

// 1MiB buffer
uint8_t buf[1024*1024];
size_t bytes;

int main(void) {
    struct rabin_t *hash;
    hash = rabin_init();

    unsigned int chunks = 0;

    while (!feof(stdin)) {
        size_t len = fread(buf, 1, sizeof(buf), stdin);
        uint8_t *ptr = &buf[0];

        bytes += len;

        while (1) {
            int remaining = rabin_next_chunk(hash, ptr, len);

            if (remaining < 0) {
                break;
            }

            len -= remaining;
            ptr += remaining;

            printf("%d %016llx\n",
                last_chunk.length,
                (long long unsigned int)last_chunk.cut_fingerprint);

            chunks++;
        }
    }

    rabin_finalize(hash);
    chunks++;

    printf("%d %016llx\n",
        last_chunk.length,
        (long long unsigned int)last_chunk.cut_fingerprint);

    unsigned int avg = 0;
    if (chunks > 0)
        avg = bytes / chunks;
    printf("%d chunks, average chunk size %d\n", chunks, avg);

    return 0;
{'fraction_non_alphanumeric': 0.14405487804878048, 'fraction_numerical': 0.023246951219512195, 'mean_word_length': 1.9395296752519597, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5148065', 'n_tokens_mistral': 949, 'n_tokens_neox': 869, 'n_words': 220}