<commit_before>#include <cstdio>
#include <cstdint>
#include <cassert>
#include <cstring>
#include <string>
#include <vector>

#include <smmintrin.h>
#if defined(HAVE_AVX2_INSTRUCTIONS) || defined(HAVE_AVX512F_INSTRUCTIONS)
#   include <immintrin.h>
#endif

// ------------------------------------------------------------------------

#include <utils/sse.cpp>
#include <utils/bits.cpp>
#include "sse4-strstr.cpp"
#ifdef HAVE_AVX2_INSTRUCTIONS
#   include <utils/avx2.cpp>
#   include "avx2-strstr.cpp"
#endif
#ifdef HAVE_AVX512F_INSTRUCTIONS
#   include "avx512f-strstr.cpp"
#endif

// ------------------------------------------------------------------------

#include <utils/ansi.cpp>
#include "application_base.cpp"


class Application final: public ApplicationBase {

public:
    bool run() {

        const auto n = words.size();

        for (size_t i = 0; i < n; i++) {

            const auto& word = words[i];

            const auto reference = file.find(word);
            const auto result_sse4 = sse4_strstr(file, word);
#ifdef HAVE_AVX2_INSTRUCTIONS
            const auto result_avx2 = avx2_strstr(file, word);
#endif
#ifdef HAVE_AVX512F_INSTRUCTIONS
            const auto result_avx512f = avx512f_strstr(file, word);
#endif
    
            if (i % 100 == 0) {
                print_progress(i, n);
            }

            if (reference != result_sse4) {
                putchar('\n');
                const auto msg = ansi::seq("ERROR", ansi::RED);
                printf("%s: std::find result = %lu, sse4_string = %lu\n",
                    msg.data(), reference, result_sse4);

                printf("word: '%s' (length %lu)\n", word.data(), word.size());

                return false;
            }

#ifdef HAVE_AVX2_INSTRUCTIONS
            if (reference != result_avx2) {
                putchar('\n');
                const auto msg = ansi::seq("ERROR", ansi::RED);
                printf("%s: std::find result = %lu, avx2_string = %lu\n",
                    msg.data(), reference, result_avx2);

                printf("word: '%s' (length %lu)\n", word.data(), word.size());

                return false;
            }
#endif

#ifdef HAVE_AVX512F_INSTRUCTIONS
            if (reference != result_avx512f) {
                putchar('\n');
                const auto msg = ansi::seq("ERROR", ansi::RED);
                printf("%s: std::find result = %lu, avx512f_string = %lu\n",
                    msg.data(), reference, result_avx512f);

                printf("word: '%s' (length %lu)\n", word.data(), word.size());

                return false;
            }
#endif
        }

        print_progress(n, n);

        putchar('\n');

        const auto msg = ansi::seq("OK", ansi::GREEN);
        printf("%s\n", msg.c_str());

        return true;
    }


    void print_help(const char* progname) {
        std::printf("usage: %s [file] [words]\n", progname);
        std::puts("");
        std::puts("Search all words in a file using std::string::find and SSE4 procedure");
        std::puts("");
        std::puts("Parameters:");
        std::puts("");<commit_msg>plug new AVX512 procedure into validate util<commit_after>#include <cstdio>
#include <cstdint>
#include <cassert>
#include <cstring>
#include <string>
#include <vector>

#include <smmintrin.h>
#if defined(HAVE_AVX2_INSTRUCTIONS) || defined(HAVE_AVX512F_INSTRUCTIONS)
#   include <immintrin.h>
#endif

// ------------------------------------------------------------------------

#include <utils/sse.cpp>
#include <utils/bits.cpp>
#include "sse4-strstr.cpp"
#ifdef HAVE_AVX2_INSTRUCTIONS
#   include <utils/avx2.cpp>
#   include "avx2-strstr.cpp"
#endif
#ifdef HAVE_AVX512F_INSTRUCTIONS
#   include "avx512f-strstr.cpp"
#   include "avx512f-strstr-v2.cpp"
#endif

// ------------------------------------------------------------------------

#include <utils/ansi.cpp>
#include "application_base.cpp"


class Application final: public ApplicationBase {

public:
    bool run() {

        const auto n = words.size();

        for (size_t i = 0; i < n; i++) {

            const auto& word = words[i];

            const auto reference = file.find(word);
            const auto result_sse4 = sse4_strstr(file, word);
#ifdef HAVE_AVX2_INSTRUCTIONS
            const auto result_avx2 = avx2_strstr(file, word);
#endif
#ifdef HAVE_AVX512F_INSTRUCTIONS
            const auto result_avx512f = avx512f_strstr(file, word);
            const auto result_avx512f_v2 = avx512f_strstr_v2(file, word);
#endif
    
            if (i % 100 == 0) {
                print_progress(i, n);
            }

            if (reference != result_sse4) {
                putchar('\n');
                const auto msg = ansi::seq("ERROR", ansi::RED);
                printf("%s: std::find result = %lu, sse4_string = %lu\n",
                    msg.data(), reference, result_sse4);

                printf("word: '%s' (length %lu)\n", word.data(), word.size());

                return false;
            }

#ifdef HAVE_AVX2_INSTRUCTIONS
            if (reference != result_avx2) {
                putchar('\n');
                const auto msg = ansi::seq("ERROR", ansi::RED);
                printf("%s: std::find result = %lu, avx2_string = %lu\n",
                    msg.data(), reference, result_avx2);

                printf("word: '%s' (length %lu)\n", word.data(), word.size());

                return false;
            }
#endif

#ifdef HAVE_AVX512F_INSTRUCTIONS
            if (reference != result_avx512f) {
                putchar('\n');
                const auto msg = ansi::seq("ERROR", ansi::RED);
                printf("%s: std::find result = %lu, avx512f_string = %lu\n",
                    msg.data(), reference, result_avx512f);

                printf("word: '%s' (length %lu)\n", word.data(), word.size());

                return false;
            }

            if (reference != result_avx512f_v2) {
                putchar('\n');
                const auto msg = ansi::seq("ERROR", ansi::RED);
                printf("%s: std::find result = %lu, avx512f_v2_string = %lu\n",
                    msg.data(), reference, result_avx512f_v2);

                printf("word: '%s' (length %lu)\n", word.data(), word.size());

                return false;
            }
#endif
        }

        print_progress(n, n);

        putchar('\n');

        const auto msg = ansi::seq("OK", ansi::GREEN);
        printf("%s\n", msg.c_str());

        return true;
    }


    void print_help(const char* progname) {
        std::printf("usage: %s [file] [words]\n", progname);
        std::puts("");
        std::puts("Search all words in a file using std::string::find and SSE4 procedure");
        std::puts("");
        std::puts("Parameters:");
        std::puts("");
{'fraction_non_alphanumeric': 0.1841988459831336, 'fraction_numerical': 0.019677467080929132, 'mean_word_length': 1.8523206751054853, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1306691', 'n_tokens_mistral': 2250, 'n_tokens_neox': 2089, 'n_words': 470}