<commit_before>#include "regex.h"
#include "unicode.h"

namespace regex {

Regex compile(const unicode& pattern) {
    return Regex(pattern.encode());
}

REMatch Regex::match(const unicode &str, uint32_t start_pos) const {
    REMatch result;

    std::locale old;
    std::locale::global(std::locale("en_US.UTF-8"));

    std::string encoded = str.encode();

    boost::match_results<std::string::const_iterator> matches;

    std::string::const_iterator start = encoded.begin();
    std::string::const_iterator finish = encoded.end();

    boost::match_flag_type flags = boost::match_default;
    flags |= boost::match_continuous;

    boost::regex_search(start + start_pos, finish, matches, _impl, flags);

    for(auto match: matches) {
        unicode match_string(match.first, match.second);

        std::string::const_iterator beg = match.first;
        std::string::const_iterator end = match.second;

        result.groups_.push_back(REMatch::Group({
            match_string,<commit_msg>Don't record matches where the beginning and end are the same (?)<commit_after>#include "regex.h"
#include "unicode.h"

namespace regex {

Regex compile(const unicode& pattern) {
    return Regex(pattern.encode());
}

REMatch Regex::match(const unicode &str, uint32_t start_pos) const {
    REMatch result;

    std::locale old;
    std::locale::global(std::locale("en_US.UTF-8"));

    std::string encoded = str.encode();

    boost::match_results<std::string::const_iterator> matches;

    std::string::const_iterator start = encoded.begin();
    std::string::const_iterator finish = encoded.end();

    boost::match_flag_type flags = boost::match_default;
    flags |= boost::match_continuous;

    boost::regex_search(start + start_pos, finish, matches, _impl, flags);

    for(auto match: matches) {
        unicode match_string(match.first, match.second);

        std::string::const_iterator beg = match.first;
        std::string::const_iterator end = match.second;

        if(beg == end) {
            continue;
        }

        result.groups_.push_back(REMatch::Group({
            match_string,
{'fraction_non_alphanumeric': 0.13437796771130103, 'fraction_numerical': 0.002849002849002849, 'mean_word_length': 3.9344262295081966, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5785724', 'n_tokens_mistral': 649, 'n_tokens_neox': 615, 'n_words': 133}