<commit_before>/**
 * @file
 * @brief Implementation of config reader
 * @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

#include "ConfigReader.hpp"

#include <algorithm>
#include <cstdlib>
#include <fstream>
#include <string>
#include <vector>

#include "core/utils/file.h"
#include "core/utils/log.h"
#include "exceptions.h"

using namespace allpix;

ConfigReader::ConfigReader() = default;
ConfigReader::ConfigReader(std::istream& stream, std::string file_name) : ConfigReader() {
    add(stream, std::move(file_name));
}

ConfigReader::ConfigReader(const ConfigReader& other) : conf_array_(other.conf_array_) {
    copy_init_map();
}
ConfigReader& ConfigReader::operator=(const ConfigReader& other) {
    conf_array_ = other.conf_array_;
    copy_init_map();
    return *this;
}

void ConfigReader::copy_init_map() {
    conf_map_.clear();
    for(auto iter = conf_array_.begin(); iter != conf_array_.end(); ++iter) {
        conf_map_[iter->getName()].push_back(iter);
    }
}

/**
 * @throws KeyValueParseError If the key / value pair could not be parsed
 *
 * The key / value pair is split according to the format specifications
 */
std::pair<std::string, std::string> ConfigReader::parseKeyValue(std::string line) {
    line = allpix::trim(line);
    size_t equals_pos = line.find('=');
    if(equals_pos != std::string::npos) {
        std::string key = trim(std::string(line, 0, equals_pos));
        std::string value = trim(std::string(line, equals_pos + 1));
        char last_quote = 0;
        for(size_t i = 0; i < value.size(); ++i) {
            if(value[i] == '\'' || value[i] == '\"') {
                if(last_quote == 0) {
                    last_quote = value[i];
                } else if(last_quote == value[i]) {
                    last_quote = 0;
                }
            }
            if(last_quote == 0 && value[i] == '#') {
                value = std::string(value, 0, i);
                break;
            }
        }

        // Check if key contains only alphanumeric or underscores
        bool valid_key = true;
        for(auto& ch : key) {
            if(isalnum(ch) == 0 && ch != '_' && ch != '.' && ch != ':') {
                valid_key = false;
                break;
            }
        }

        // Check if value is not empty and key is valid
        if(!valid_key) {
            throw KeyValueParseError(line, "key is not valid");
        }
        if(value.empty()) {
            throw KeyValueParseError(line, "value is empty");
        }

        return std::make_pair(key, allpix::trim(value));
    }
    // Key / value pair does not contain equal sign
    throw KeyValueParseError(line, "missing equality sign to split key and value");
}

/**
 * @throws ConfigParseError If an error occurred during the parsing of the stream
 *
 * The configuration is immediately parsed and all of its configurations are available after the functions returns.
 */
void ConfigReader::add(std::istream& stream, std::string file_name) {
    LOG(TRACE) << "Parsing configuration file " << file_name;

    // Convert file name to absolute path (if given)
    if(!file_name.empty()) {
        file_name = allpix::get_absolute_path(file_name);
    }

    // Build first empty configuration
    std::string section_name;
    Configuration conf(section_name, file_name);

    int line_num = 0;
    while(true) {
        // Process config line by line
        std::string line;
        if(stream.eof()) {
            break;
        }
        std::getline(stream, line);
        ++line_num;

        // Ignore empty lines or comments
        if(line.empty() || line.front() == '#') {
            continue;
        }

        // Check if section header or key-value pair
        if(line.front() == '[') {
            // Line should be a section header with an alphanumeric name
            size_t idx = 1;
            for(; idx < line.length() - 1; ++idx) {
                if(isalnum(line[idx]) == 0 && line[idx] != '_') {
                    break;
                }
            }
            std::string remain = allpix::trim(line.substr(idx + 1));
            if(line[idx] == ']' && (remain.empty() || remain.front() == '#')) {
                // Ignore empty sections if they contain no configurations<commit_msg>Fix bug: user manual states that all whitespace at beginning & end shall be stripped. Do that.<commit_after>/**
 * @file
 * @brief Implementation of config reader
 * @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

#include "ConfigReader.hpp"

#include <algorithm>
#include <cstdlib>
#include <fstream>
#include <string>
#include <vector>

#include "core/utils/file.h"
#include "core/utils/log.h"
#include "exceptions.h"

using namespace allpix;

ConfigReader::ConfigReader() = default;
ConfigReader::ConfigReader(std::istream& stream, std::string file_name) : ConfigReader() {
    add(stream, std::move(file_name));
}

ConfigReader::ConfigReader(const ConfigReader& other) : conf_array_(other.conf_array_) {
    copy_init_map();
}
ConfigReader& ConfigReader::operator=(const ConfigReader& other) {
    conf_array_ = other.conf_array_;
    copy_init_map();
    return *this;
}

void ConfigReader::copy_init_map() {
    conf_map_.clear();
    for(auto iter = conf_array_.begin(); iter != conf_array_.end(); ++iter) {
        conf_map_[iter->getName()].push_back(iter);
    }
}

/**
 * @throws KeyValueParseError If the key / value pair could not be parsed
 *
 * The key / value pair is split according to the format specifications
 */
std::pair<std::string, std::string> ConfigReader::parseKeyValue(std::string line) {
    line = allpix::trim(line);
    size_t equals_pos = line.find('=');
    if(equals_pos != std::string::npos) {
        std::string key = trim(std::string(line, 0, equals_pos));
        std::string value = trim(std::string(line, equals_pos + 1));
        char last_quote = 0;
        for(size_t i = 0; i < value.size(); ++i) {
            if(value[i] == '\'' || value[i] == '\"') {
                if(last_quote == 0) {
                    last_quote = value[i];
                } else if(last_quote == value[i]) {
                    last_quote = 0;
                }
            }
            if(last_quote == 0 && value[i] == '#') {
                value = std::string(value, 0, i);
                break;
            }
        }

        // Check if key contains only alphanumeric or underscores
        bool valid_key = true;
        for(auto& ch : key) {
            if(isalnum(ch) == 0 && ch != '_' && ch != '.' && ch != ':') {
                valid_key = false;
                break;
            }
        }

        // Check if value is not empty and key is valid
        if(!valid_key) {
            throw KeyValueParseError(line, "key is not valid");
        }
        if(value.empty()) {
            throw KeyValueParseError(line, "value is empty");
        }

        return std::make_pair(key, allpix::trim(value));
    }
    // Key / value pair does not contain equal sign
    throw KeyValueParseError(line, "missing equality sign to split key and value");
}

/**
 * @throws ConfigParseError If an error occurred during the parsing of the stream
 *
 * The configuration is immediately parsed and all of its configurations are available after the functions returns.
 */
void ConfigReader::add(std::istream& stream, std::string file_name) {
    LOG(TRACE) << "Parsing configuration file " << file_name;

    // Convert file name to absolute path (if given)
    if(!file_name.empty()) {
        file_name = allpix::get_absolute_path(file_name);
    }

    // Build first empty configuration
    std::string section_name;
    Configuration conf(section_name, file_name);

    int line_num = 0;
    while(true) {
        // Process config line by line
        std::string line;
        if(stream.eof()) {
            break;
        }
        std::getline(stream, line);
        ++line_num;

        // Trim whitespaces at beginning and end of line:
        line = allpix::trim(line);

        // Ignore empty lines or comments
        if(line.empty() || line.front() == '#') {
            continue;
        }

        // Check if section header or key-value pair
        if(line.front() == '[') {
            // Line should be a section header with an alphanumeric name
            size_t idx = 1;
            for(; idx < line.length() - 1; ++idx) {
                if(isalnum(line[idx]) == 0 && line[idx] != '_') {
                    break;
                }
            }
            std::string remain = allpix::trim(line.substr(idx + 1));
            if(line[idx] == ']' && (remain.empty() || remain.front() == '#')) {
                // Ignore empty sections if they contain no configurations
{'fraction_non_alphanumeric': 0.12948484523937026, 'fraction_numerical': 0.0038556281460854665, 'mean_word_length': 2.395636363636364, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7536003', 'n_tokens_mistral': 2760, 'n_tokens_neox': 2654, 'n_words': 788}