<commit_before>/*
 * util.cpp
 */

#include "util.hpp"

#include <regex>
#include <cassert>
#include <numeric>
#include <random>
#include <cstring>

#include <sys/mman.h>


std::string escape_for_regex(const std::string& input) {
    // see https://stackoverflow.com/a/40195721/149138
    static std::regex chars_to_escape{ R"([-[\]{}()*+?.\^$|])" };
    return std::regex_replace( input, chars_to_escape, R"(\$&)" );
}

bool wildcard_match(const std::string& target, const std::string& pattern) {
    // we implement wildcards just by replacing all *s with .*
    std::string escaped = escape_for_regex(std::move(pattern));
    std::string regex   = std::regex_replace(escaped, std::regex(R"(\\\*)"), R"(.*)");
    return std::regex_match(target, std::regex(regex));
}

const size_t TWO_MB = 2 * 1024 * 1024;
const int STORAGE_SIZE = 100 * 1024 * 1024;  // 100 MB
void *storage_ptr = 0;

volatile int zero = 0;
bool storage_init = false;

/**
 * Return a new pointer to a memory region of at least size, aligned to a 2MB boundary and with<commit_msg>add support for pre-4.9 using a fallback wildcard search<commit_after>/*
 * util.cpp
 */

#include "util.hpp"

#include <regex>
#include <cassert>
#include <numeric>
#include <random>
#include <cstring>
#include <exception>

#include <sys/mman.h>

#if defined(__GNUC__) && !defined(__clang__)
#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
#endif

#if defined(GCC_VERSION) && (GCC_VERSION < 40900)
#warning "Your gcc is too old to support regex, wildcard matches will be disabled - use gcc 4.9 or higher if you need them"
#define NO_CXX11_REGEX
#endif

#ifdef NO_CXX11_REGEX

// support only a basic form of wildcard matches with works with literal patterns (no wildcards)
// or a single trailing wildcard
bool wildcard_match(const std::string& target, const std::string& pattern) {
    auto first_star = pattern.find('*');
    if (first_star == std::string::npos) {
        return target == pattern;
    } else if (first_star == pattern.length() - 1) {
        std::string prefix = pattern.substr(0, pattern.length() - 1);
        return target.find(prefix) == 0;
    } else {
        throw std::runtime_error("You tried to use a non-trailing wildcard match, but this feature that requires C++ regex support");
    }
}

#else

std::string escape_for_regex(const std::string& input) {
    // see https://stackoverflow.com/a/40195721/149138
    static std::regex chars_to_escape{ R"([-[\]{}()*+?.\^$|])" };
    return std::regex_replace( input, chars_to_escape, R"(\$&)" );
}

bool wildcard_match(const std::string& target, const std::string& pattern) {
    // we implement wildcards just by replacing all *s with .*
    std::string escaped = escape_for_regex(std::move(pattern));
    std::string regex   = std::regex_replace(escaped, std::regex(R"(\\\*)"), R"(.*)");
    return std::regex_match(target, std::regex(regex));
}

#endif

const size_t TWO_MB = 2 * 1024 * 1024;
const int STORAGE_SIZE = 100 * 1024 * 1024;  // 100 MB
void *storage_ptr = 0;

volatile int zero = 0;
bool storage_init = false;

/**
 * Return a new pointer to a memory region of at least size, aligned to a 2MB boundary and with
{'fraction_non_alphanumeric': 0.14890282131661442, 'fraction_numerical': 0.032915360501567396, 'mean_word_length': 4.180194805194805, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4748661', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1047, 'n_words': 323}