<commit_before>
#ifdef __cplusplus
#define __STDC_LIMIT_MACROS
#define __STDC_CONSTANT_MACROS
#define __STDC_FORMAT_MACROS
#endif

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <inttypes.h>
#include <memory>

#include <type_traits>
#include <iostream>

#include "strlen_fast.h"
#include "stop_watch.h"

enum {
    test_fixed_string,
    test_fixed_length,
    test_random_length,
    test_type_max
};

void generate_random_string(char * str, size_t str_len)
{
    unsigned char * dest = (unsigned char *)str;
    for (size_t i = 0; i < str_len; ++i)
        *dest++ = (unsigned char)(rand() % 255) + 1;
    *dest = '\0';
}

void generate_random_string_array(char * str, size_t length, size_t str_len, size_t alloc_size)
{
    unsigned char * dest = (unsigned char *)str;
    memset(dest, 'a', alloc_size);
    for (size_t i = 0; i < length; ++i) {
        //unsigned char * cur = (unsigned char *)dest;
        //for (size_t j = 0; j < str_len; ++j)
        //    *cur++ = (unsigned char)(rand() % 255) + 1;
        size_t base = str_len / 2;
        size_t len = base + rand() % (str_len - base);
        *(dest + len) = '\0';
        dest += str_len;
    }
}

void strlen_benchmark_fixed_string(uint32_t str_len, uint32_t iterations)
{
    if (str_len <= 0) {
        printf("Type: test_fixed_string, Error: str_len <= 0, str_len = %u\n\n",
            str_len);
        return;
    }

    printf("Type: test_fixed_string, iterations = %u, str_len = %u byte(s)\n\n",
        iterations, str_len);

    {
        stop_watch sw;
        size_t sum = 0;
        size_t len;
        int dir = -1;
        (void)dir;
        std::unique_ptr<char> _str(new char[str_len]);
        char * str = _str.get();
        generate_random_string(str, str_len - 1);

        sw.start();
        for (uint32_t i = 0; i < iterations; ++i) {
            len = ::strlen(str);
#if defined(__GUNC__)
            if ((len == 0 && dir == -1) || (len == (str_len - 1) && dir == 1))
                dir = -dir;
            str += dir;
#endif
            sum += len;
        }
        sw.stop();

        str = _str.get();
        printf("%30s = %8" PRIuPTR ", time spent: %8.3f ms, sum: %" PRIuPTR "\n",
            "::strlen(str)", ::strlen(str), sw.getMillisec(), sum);
    }

    {
        stop_watch sw;
        size_t sum = 0;
        size_t len;
        std::unique_ptr<char> _str(new char[str_len]);
        char * str = _str.get();
        generate_random_string(str, str_len - 1);

        sw.start();
        for (uint32_t i = 0; i < iterations; ++i) {
            len = strlen_fast_v1_sse2(str);
            sum += len;
        }
        sw.stop();

        printf("%30s = %8" PRIuPTR ", time spent: %8.3f ms, sum: %" PRIuPTR "\n",
            "strlen_fast_v1_sse2(str)", strlen_fast_v1_sse2(str), sw.getMillisec(), sum);<commit_msg>修正一处 __GNUC__ 宏的定义错误;<commit_after>
#ifdef __cplusplus
#define __STDC_LIMIT_MACROS
#define __STDC_CONSTANT_MACROS
#define __STDC_FORMAT_MACROS
#endif

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <inttypes.h>
#include <memory>

#include <type_traits>
#include <iostream>

#include "strlen_fast.h"
#include "stop_watch.h"

enum {
    test_fixed_string,
    test_fixed_length,
    test_random_length,
    test_type_max
};

void generate_random_string(char * str, size_t str_len)
{
    unsigned char * dest = (unsigned char *)str;
    for (size_t i = 0; i < str_len; ++i)
        *dest++ = (unsigned char)(rand() % 255) + 1;
    *dest = '\0';
}

void generate_random_string_array(char * str, size_t length, size_t str_len, size_t alloc_size)
{
    unsigned char * dest = (unsigned char *)str;
    memset(dest, 'a', alloc_size);
    for (size_t i = 0; i < length; ++i) {
        //unsigned char * cur = (unsigned char *)dest;
        //for (size_t j = 0; j < str_len; ++j)
        //    *cur++ = (unsigned char)(rand() % 255) + 1;
        size_t base = str_len / 2;
        size_t len = base + rand() % (str_len - base);
        *(dest + len) = '\0';
        dest += str_len;
    }
}

void strlen_benchmark_fixed_string(uint32_t str_len, uint32_t iterations)
{
    if (str_len <= 0) {
        printf("Type: test_fixed_string, Error: str_len <= 0, str_len = %u\n\n",
            str_len);
        return;
    }

    printf("Type: test_fixed_string, iterations = %u, str_len = %u byte(s)\n\n",
        iterations, str_len);

    {
        stop_watch sw;
        size_t sum = 0;
        size_t len;
        int dir = -1;
        (void)dir;
        std::unique_ptr<char> _str(new char[str_len]);
        char * str = _str.get();
        generate_random_string(str, str_len - 1);

        sw.start();
        for (uint32_t i = 0; i < iterations; ++i) {
            len = ::strlen(str);
#if defined(__GNUC__)
            if ((len == 0 && dir == -1) || (len == (str_len - 1) && dir == 1))
                dir = -dir;
            str += dir;
#endif
            sum += len;
        }
        sw.stop();

        str = _str.get();
        printf("%30s = %8" PRIuPTR ", time spent: %8.3f ms, sum: %" PRIuPTR "\n",
            "::strlen(str)", ::strlen(str), sw.getMillisec(), sum);
    }

    {
        stop_watch sw;
        size_t sum = 0;
        size_t len;
        std::unique_ptr<char> _str(new char[str_len]);
        char * str = _str.get();
        generate_random_string(str, str_len - 1);

        sw.start();
        for (uint32_t i = 0; i < iterations; ++i) {
            len = strlen_fast_v1_sse2(str);
            sum += len;
        }
        sw.stop();

        printf("%30s = %8" PRIuPTR ", time spent: %8.3f ms, sum: %" PRIuPTR "\n",
            "strlen_fast_v1_sse2(str)", strlen_fast_v1_sse2(str), sw.getMillisec(), sum);
{'fraction_non_alphanumeric': 0.15372452565003514, 'fraction_numerical': 0.017919887561489812, 'mean_word_length': 2.251284980011422, 'pattern_counts': {'":': 2, '<': 35, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6861984', 'n_tokens_mistral': 2171, 'n_tokens_neox': 2100, 'n_words': 503}