<commit_before>#include <cstdio>
#include <cstdint>
#include <cstring>
#include <vector>
#include <algorithm>

#include <sys/time.h>

static inline int ascii_std(const uint8_t *data, int len)
{
    return !std::any_of(data, data+len, [] (int8_t b) { return b < 0; });
}

static inline int ascii_u64(const uint8_t *data, int len)
{
   uint8_t orall = 0;

    if (len >= 16) {
        union {
            uint64_t u64;
            uint8_t u8[8];
        } orr;

        uint64_t or1 = 0, or2 = 0;
        const uint8_t *data2 = data+8;

        while (len >= 16) {
            or1 |= *(const uint64_t *)data;
            or2 |= *(const uint64_t *)data2;
            data += 16;
            data2 += 16;
            len -= 16;
        }

        orr.u64 = or1 | or2;
        for (int i = 0; i < 8; ++i)
            orall |= orr.u8[i];
    }

    while (len--)
        orall |= *data++;

    return orall < 0x80;
}

#if defined(__x86_64__)
#include <x86intrin.h>

static inline int ascii_simd(const uint8_t *data, int len)
{
    if (len >= 32) {
        const uint8_t *data1 = data, *data2 = data+16;

        __m128i or1 = _mm_set1_epi8(0), or2 = or1;

        while (len >= 32) {
            __m128i input1 = _mm_lddqu_si128((const __m128i *)data1);
            __m128i input2 = _mm_lddqu_si128((const __m128i *)data2);

            or1 = _mm_or_si128(or1, input1);
            or2 = _mm_or_si128(or2, input2);

            data1 += 32;
            data2 += 32;
            len -= 32;
        }

        or1 = _mm_or_si128(or1, or2);
        if (_mm_movemask_epi8(_mm_cmplt_epi8(or1, _mm_set1_epi8(0))))
            return 0;

        data = data1;
    }

    return ascii_u64(data, len);
}

#elif defined(__aarch64__)
#include <arm_neon.h>

static inline int ascii_simd(const uint8_t *data, int len)
{
    if (len >= 32) {
        const uint8_t *data1 = data, *data2 = data+16;

        uint8x16_t or1 = vdupq_n_u8(0), or2 = or1;

        while (len >= 32) {
            const uint8x16_t input1 = vld1q_u8(data1);
            const uint8x16_t input2 = vld1q_u8(data2);

            or1 = vorrq_u8(or1, input1);
            or2 = vorrq_u8(or2, input2);

            data1 += 32;
            data2 += 32;
            len -= 32;
        }

        or1 = vorrq_u8(or1, or2);
        if (vmaxvq_u8(or1) >= 0x80)
            return 0;

        data = data1;
    }

    return ascii_u64(data, len);
}

#endif

struct ftab {
    const char *name;
    int (*func)(const uint8_t *data, int len);
};

static const std::vector<ftab> _f = {
    {
        .name = "std",
        .func = ascii_std,
    }, {
        .name = "u64",
        .func = ascii_u64,
    }, {
        .name = "simd",<commit_msg>ascii.cpp: minor refinement<commit_after>#include <cstdio>
#include <cstdint>
#include <cstring>
#include <vector>
#include <algorithm>

#include <sys/time.h>

static inline int ascii_std(const uint8_t *data, int len)
{
    return !std::any_of(data, data+len, [] (int8_t b) { return b < 0; });
}

static inline int ascii_u64(const uint8_t *data, int len)
{
    uint8_t orall = 0;

    if (len >= 16) {
        union {
            uint64_t u64;
            uint32_t u32[2];
            uint16_t u16[4];
            uint8_t u8[8];
        } orr;

        uint64_t or1 = 0, or2 = 0;
        const uint8_t *data2 = data+8;

        while (len >= 16) {
            or1 |= *(const uint64_t *)data;
            or2 |= *(const uint64_t *)data2;
            data += 16;
            data2 += 16;
            len -= 16;
        }

        orr.u64 = or1 | or2;
        orr.u32[0] |= orr.u32[1];
        orr.u16[0] |= orr.u16[1];
        orall = orr.u8[0] | orr.u8[1];
    }

    while (len--)
        orall |= *data++;

    return orall < 0x80;
}

#if defined(__x86_64__)
#include <x86intrin.h>

static inline int ascii_simd(const uint8_t *data, int len)
{
    if (len >= 32) {
        const uint8_t *data2 = data+16;

        __m128i or1 = _mm_set1_epi8(0), or2 = or1;

        while (len >= 32) {
            __m128i input1 = _mm_lddqu_si128((const __m128i *)data);
            __m128i input2 = _mm_lddqu_si128((const __m128i *)data2);

            or1 = _mm_or_si128(or1, input1);
            or2 = _mm_or_si128(or2, input2);

            data += 32;
            data2 += 32;
            len -= 32;
        }

        or1 = _mm_or_si128(or1, or2);
        if (_mm_movemask_epi8(_mm_cmplt_epi8(or1, _mm_set1_epi8(0))))
            return 0;
    }

    return ascii_u64(data, len);
}

#elif defined(__aarch64__)
#include <arm_neon.h>

static inline int ascii_simd(const uint8_t *data, int len)
{
    if (len >= 32) {
        const uint8_t *data2 = data+16;

        uint8x16_t or1 = vdupq_n_u8(0), or2 = or1;

        while (len >= 32) {
            const uint8x16_t input1 = vld1q_u8(data);
            const uint8x16_t input2 = vld1q_u8(data2);

            or1 = vorrq_u8(or1, input1);
            or2 = vorrq_u8(or2, input2);

            data += 32;
            data2 += 32;
            len -= 32;
        }

        or1 = vorrq_u8(or1, or2);
        if (vmaxvq_u8(or1) >= 0x80)
            return 0;
    }

    return ascii_u64(data, len);
}

#endif

struct ftab {
    const char *name;
    int (*func)(const uint8_t *data, int len);
};

static const std::vector<ftab> _f = {
    {
        .name = "std",
        .func = ascii_std,
    }, {
        .name = "u64",
        .func = ascii_u64,
    }, {
        .name = "simd",
{'fraction_non_alphanumeric': 0.134159881569208, 'fraction_numerical': 0.07790525536639527, 'mean_word_length': 1.8048780487804879, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '552095', 'n_tokens_mistral': 2361, 'n_tokens_neox': 2120, 'n_words': 485}