<commit_before>#pragma once

#include "config.hpp"

namespace blackhole {

namespace formatter {

namespace string {

struct pattern_parser_t {
    static config_t parse(const std::string& pattern) {
        auto current = pattern.begin();
        auto end = pattern.end();

        std::string fpattern;
        fpattern.reserve(pattern.length());
        std::vector<std::string> attribute_names;

        while(current != end) {
            if ((*current == '%') && (current + 1 != end) && (*(current + 1) == '(')) {
                fpattern.push_back('%');
                current += 2;
                std::string key;
                while (current != end) {
                    if ((*current == ')') && (current + 1 != end) && (*(current + 1) == 's')) {
                        break;
                    } else {
                        key.push_back(*current);
                    }
                    current++;
                }
                if (boost::starts_with(key, "...")) {
                    for (auto it = key.begin() + 3; it != key.end(); ++it) {
                        char ch = *it;
                        switch (ch) {
                        case 'L':
                            *it = '0';
                            break;
                        default:
                            *it = '0';
                        }
                    }
                }
                attribute_names.push_back(key);
            } else {
                fpattern.push_back(*current);
            }
            current++;
        }

        return { fpattern, attribute_names };
    }
};

} // namespace string

} // namespace formatter

} // namespace blackhole<commit_msg>Pattern parsing class has been simplified for understanding.<commit_after>#pragma once

#include <boost/algorithm/string.hpp>

#include "config.hpp"

namespace blackhole {

namespace formatter {

namespace string {

static const char VARIADIC_KEY_PREFIX[] = "...";

struct pattern_parser_t {
    static config_t parse(const std::string& input_pattern) {
        auto current = input_pattern.begin();
        auto end = input_pattern.end();

        std::string pattern;
        pattern.reserve(input_pattern.length());
        std::vector<std::string> keys;

        while(current != end) {
            if (begin_key(current, end)) {
                current += 2;
                pattern.push_back('%');
                std::string key;
                while (current != end) {
                    if (end_key(current, end)) {
                        break;
                    } else {
                        key.push_back(*current);
                    }
                    current++;
                }

                if (boost::starts_with(key, VARIADIC_KEY_PREFIX)) {
                    handle_variadic_key(&key);
                }
                keys.push_back(key);
            } else {
                pattern.push_back(*current);
            }

            current++;
        }

        return { pattern, keys };
    }

private:
    static inline bool begin_key(std::string::const_iterator it, std::string::const_iterator end) {
        return (*it == '%') && (it + 1 != end) && (*(it + 1) == '(');
    }

    static inline bool end_key(std::string::const_iterator it, std::string::const_iterator end) {
        return (*it == ')') && (it + 1 != end) && (*(it + 1) == 's');
    }

    static void handle_variadic_key(std::string* key) {
        for (auto it = key->begin() + 3; it != key->end(); ++it) {
            char ch = *it;
            switch (ch) {
            case 'L':
                *it = '0';
                break;
            default:
                *it = '0';
            }
        }
    }
};

} // namespace string

} // namespace formatter

} // namespace blackhole
{'fraction_non_alphanumeric': 0.13563339478535685, 'fraction_numerical': 0.004213853041875165, 'mean_word_length': 1.3144424131627057, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1678031', 'n_tokens_mistral': 1056, 'n_tokens_neox': 988, 'n_words': 225}