<commit_before>//gcc -msse4.2 -Wall -Wextra crchw.c

#include <stdio.h>
#include <nmmintrin.h>

int main ()
{
    unsigned int crc = 1;
    unsigned char input[] = {
    0x47, 0, 4, 42, 0x01, 0x02, 0x3, 0x4,
    };
    int n = sizeof(input)/sizeof(input[0]);
    for (int i = 0; i < n; i++) {
      crc = _mm_crc32_u8(crc, input[i]);
    }
    printf("CRC: %08x\n", crc);

    crc = 1;
    n /= 4;
    for (int i = 0; i < n; i++) {
      unsigned int* ptr = (unsigned int*)&input[i*4];
      crc = _mm_crc32_u32(crc, *ptr);
    }
    printf("CRC: %08x\n", crc);

    return 0;
}<commit_msg>[c] update CRC HW<commit_after>//gcc -msse4.2 -Wall -Wextra crchw.c

#include <stdio.h>
#include <nmmintrin.h>
#include <assert.h>

int main ()
{
    unsigned int crc1 = 1;
    unsigned char input[] = {
    0x47, 0, 4, 42, 0x01, 0x02, 0x3, 0x4,
    };
    int n = sizeof(input)/sizeof(input[0]);
    for (int i = 0; i < n; i++) {
      crc1 = _mm_crc32_u8(crc1, input[i]);
    }
    //printf("CRC: %08x\n", crc1);
    assert(crc1 == 0xd42a22cb);

    unsigned int crc2 = 1;
    assert(n % 4 == 0);
    for (int i = 0; i < n; i += 4) {
      unsigned int* ptr = (unsigned int*)&input[i];
      crc2 = _mm_crc32_u32(crc2, *ptr);
    }
    //printf("CRC: %08x\n", crc2);
    assert(crc2 == 0xd42a22cb);
    puts("pass");
    return 0;
}
{'fraction_non_alphanumeric': 0.1927437641723356, 'fraction_numerical': 0.07331821617535904, 'mean_word_length': 2.4123711340206184, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5720282', 'n_tokens_mistral': 661, 'n_tokens_neox': 597, 'n_words': 146}