<commit_before>#include <exception>
#include <iostream>
#include <sstream>
#include <boost/algorithm/string.hpp>
#include <boost/property_tree/xml_parser.hpp>
#include "pqrs/string.hpp"
#include "pqrs/xml_compiler.hpp"

namespace pqrs {
  void
  xml_compiler::reload(void)
  {
    error_information_.clear();
    replacement_.clear();
    symbol_map_.clear();
    app_vector_.clear();
    identifier_map_.clear();
    essential_configurations_.clear();
    remapclasses_initialize_vector_.clear();
    simultaneous_keycode_index_ = 0;
    preferences_checkbox_node_tree_.clear();
    preferences_number_node_tree_.clear();

    try {
      // ------------------------------------------------------------
      // Loading replacement
      {
        replacement_loader loader(*this, replacement_);

        // private.xml
        {
          ptree_ptr ptree_ptr;
          pqrs::string::replacement dummy; // Use dummy replacement when we read <replacementdef>.
          read_xml_(ptree_ptr,
                    xml_file_path(xml_file_path::base_directory::private_xml, "private.xml"),
                    dummy);
          if (ptree_ptr) {
            loader.traverse(make_extracted_ptree(*ptree_ptr));
          }
        }
        // replacementdef.xml
        {
          ptree_ptr ptree_ptr;
          pqrs::string::replacement dummy; // Use dummy replacement when we read <replacementdef>.
          read_xml_(ptree_ptr,
                    xml_file_path(xml_file_path::base_directory::system_xml,  "replacementdef.xml"),
                    dummy);
          if (ptree_ptr) {
            loader.traverse(make_extracted_ptree(*ptree_ptr));
          }
        }
      }

      // ------------------------------------------------------------
      // Then, we read private.xml with replacement and loaders share it.

      ptree_ptr private_xml_ptree_ptr;
      read_xml_(private_xml_ptree_ptr,
                xml_file_path(xml_file_path::base_directory::private_xml, "private.xml"));
      if (private_xml_ptree_ptr && private_xml_ptree_ptr->empty()) {
        private_xml_ptree_ptr.reset();
      }

      // symbol_map
      {
        symbol_map_loader loader(*this, symbol_map_);

        if (private_xml_ptree_ptr) {
          loader.traverse(make_extracted_ptree(*private_xml_ptree_ptr));
        }

        loader_wrapper<symbol_map_loader>::traverse_system_xml(*this, loader, "symbol_map.xml");
      }

      // app
      {
        app_loader loader(*this, symbol_map_, app_vector_);

        if (private_xml_ptree_ptr) {
          loader.traverse(make_extracted_ptree(*private_xml_ptree_ptr));
        }

        loader_wrapper<app_loader>::traverse_system_xml(*this, loader, "appdef.xml");
      }

      // device
      {
        device_loader loader(*this, symbol_map_);

        if (private_xml_ptree_ptr) {
          loader.traverse(make_extracted_ptree(*private_xml_ptree_ptr));
        }

        loader_wrapper<device_loader>::traverse_system_xml(*this, loader, "devicevendordef.xml");
        loader_wrapper<device_loader>::traverse_system_xml(*this, loader, "deviceproductdef.xml");
      }

      // config_index, remapclasses_initialize_vector, preferences_node
      {
        ptree_ptr checkbox_xml_ptree_ptr;
        read_xml_(checkbox_xml_ptree_ptr,
                  xml_file_path(xml_file_path::base_directory::system_xml, "checkbox.xml"));

        ptree_ptr number_xml_ptree_ptr;
        read_xml_(number_xml_ptree_ptr,
                  xml_file_path(xml_file_path::base_directory::system_xml, "number.xml"));

        // remapclasses_initialize_vector
        {
          // prepare
          {
            remapclasses_initialize_vector_prepare_loader<preferences_node_tree<preferences_number_node> > loader(*this, symbol_map_, identifier_map_, essential_configurations_, &preferences_number_node_tree_);

            if (number_xml_ptree_ptr) {
              loader.traverse(make_extracted_ptree(*number_xml_ptree_ptr));
              loader.fixup();
            }
            loader.cleanup();
          }
          {
            remapclasses_initialize_vector_prepare_loader<preferences_node_tree<preferences_checkbox_node> > loader(*this, symbol_map_, identifier_map_, essential_configurations_, &preferences_checkbox_node_tree_);

            if (private_xml_ptree_ptr) {
              loader.traverse(make_extracted_ptree(*private_xml_ptree_ptr));
              loader.fixup();
            }
            if (checkbox_xml_ptree_ptr) {
              loader.traverse(make_extracted_ptree(*checkbox_xml_ptree_ptr));
              loader.fixup();
            }
            loader.cleanup();
          }

          // ----------------------------------------
          if (private_xml_ptree_ptr) {
            traverse_identifier_(make_extracted_ptree(*private_xml_ptree_ptr), "");
          }
          if (checkbox_xml_ptree_ptr) {
            traverse_identifier_(make_extracted_ptree(*checkbox_xml_ptree_ptr), "");
          }

          remapclasses_initialize_vector_.freeze();
        }
      }

    } catch (std::exception& e) {
      error_information_.set(e.what());
    }
  }

  void
  xml_compiler::read_xml_(ptree_ptr& out,
                          const std::string& base_diretory,
                          const std::string& relative_file_path,
                          const pqrs::string::replacement& replacement) const
  {
    try {
      out.reset(new boost::property_tree::ptree());

      std::string path = base_diretory + "/" + relative_file_path;

      std::string xml;
      if (replacement.empty()) {
        pqrs::string::string_from_file(xml, path.c_str());
      } else {
        pqrs::string::string_by_replacing_double_curly_braces_from_file(xml, path.c_str(), replacement);
      }
      if (xml.empty()) {
        error_information_.set(relative_file_path + " is not found.");
        return;
      }

      std::stringstream istream(xml, std::stringstream::in);

      int flags = boost::property_tree::xml_parser::no_comments;

      boost::property_tree::read_xml(istream, *out, flags);

    } catch (std::exception& e) {
      std::string what = e.what();<commit_msg>absolute path support at include tag<commit_after>#include <exception>
#include <iostream>
#include <sstream>
#include <boost/algorithm/string.hpp>
#include <boost/property_tree/xml_parser.hpp>
#include "pqrs/string.hpp"
#include "pqrs/xml_compiler.hpp"

namespace pqrs {
  void
  xml_compiler::reload(void)
  {
    error_information_.clear();
    replacement_.clear();
    symbol_map_.clear();
    app_vector_.clear();
    identifier_map_.clear();
    essential_configurations_.clear();
    remapclasses_initialize_vector_.clear();
    simultaneous_keycode_index_ = 0;
    preferences_checkbox_node_tree_.clear();
    preferences_number_node_tree_.clear();

    try {
      // ------------------------------------------------------------
      // Loading replacement
      {
        replacement_loader loader(*this, replacement_);

        // private.xml
        {
          ptree_ptr ptree_ptr;
          pqrs::string::replacement dummy; // Use dummy replacement when we read <replacementdef>.
          read_xml_(ptree_ptr,
                    xml_file_path(xml_file_path::base_directory::private_xml, "private.xml"),
                    dummy);
          if (ptree_ptr) {
            loader.traverse(make_extracted_ptree(*ptree_ptr));
          }
        }
        // replacementdef.xml
        {
          ptree_ptr ptree_ptr;
          pqrs::string::replacement dummy; // Use dummy replacement when we read <replacementdef>.
          read_xml_(ptree_ptr,
                    xml_file_path(xml_file_path::base_directory::system_xml,  "replacementdef.xml"),
                    dummy);
          if (ptree_ptr) {
            loader.traverse(make_extracted_ptree(*ptree_ptr));
          }
        }
      }

      // ------------------------------------------------------------
      // Then, we read private.xml with replacement and loaders share it.

      ptree_ptr private_xml_ptree_ptr;
      read_xml_(private_xml_ptree_ptr,
                xml_file_path(xml_file_path::base_directory::private_xml, "private.xml"));
      if (private_xml_ptree_ptr && private_xml_ptree_ptr->empty()) {
        private_xml_ptree_ptr.reset();
      }

      // symbol_map
      {
        symbol_map_loader loader(*this, symbol_map_);

        if (private_xml_ptree_ptr) {
          loader.traverse(make_extracted_ptree(*private_xml_ptree_ptr));
        }

        loader_wrapper<symbol_map_loader>::traverse_system_xml(*this, loader, "symbol_map.xml");
      }

      // app
      {
        app_loader loader(*this, symbol_map_, app_vector_);

        if (private_xml_ptree_ptr) {
          loader.traverse(make_extracted_ptree(*private_xml_ptree_ptr));
        }

        loader_wrapper<app_loader>::traverse_system_xml(*this, loader, "appdef.xml");
      }

      // device
      {
        device_loader loader(*this, symbol_map_);

        if (private_xml_ptree_ptr) {
          loader.traverse(make_extracted_ptree(*private_xml_ptree_ptr));
        }

        loader_wrapper<device_loader>::traverse_system_xml(*this, loader, "devicevendordef.xml");
        loader_wrapper<device_loader>::traverse_system_xml(*this, loader, "deviceproductdef.xml");
      }

      // config_index, remapclasses_initialize_vector, preferences_node
      {
        ptree_ptr checkbox_xml_ptree_ptr;
        read_xml_(checkbox_xml_ptree_ptr,
                  xml_file_path(xml_file_path::base_directory::system_xml, "checkbox.xml"));

        ptree_ptr number_xml_ptree_ptr;
        read_xml_(number_xml_ptree_ptr,
                  xml_file_path(xml_file_path::base_directory::system_xml, "number.xml"));

        // remapclasses_initialize_vector
        {
          // prepare
          {
            remapclasses_initialize_vector_prepare_loader<preferences_node_tree<preferences_number_node> > loader(*this, symbol_map_, identifier_map_, essential_configurations_, &preferences_number_node_tree_);

            if (number_xml_ptree_ptr) {
              loader.traverse(make_extracted_ptree(*number_xml_ptree_ptr));
              loader.fixup();
            }
            loader.cleanup();
          }
          {
            remapclasses_initialize_vector_prepare_loader<preferences_node_tree<preferences_checkbox_node> > loader(*this, symbol_map_, identifier_map_, essential_configurations_, &preferences_checkbox_node_tree_);

            if (private_xml_ptree_ptr) {
              loader.traverse(make_extracted_ptree(*private_xml_ptree_ptr));
              loader.fixup();
            }
            if (checkbox_xml_ptree_ptr) {
              loader.traverse(make_extracted_ptree(*checkbox_xml_ptree_ptr));
              loader.fixup();
            }
            loader.cleanup();
          }

          // ----------------------------------------
          if (private_xml_ptree_ptr) {
            traverse_identifier_(make_extracted_ptree(*private_xml_ptree_ptr), "");
          }
          if (checkbox_xml_ptree_ptr) {
            traverse_identifier_(make_extracted_ptree(*checkbox_xml_ptree_ptr), "");
          }

          remapclasses_initialize_vector_.freeze();
        }
      }

    } catch (std::exception& e) {
      error_information_.set(e.what());
    }
  }

  void
  xml_compiler::read_xml_(ptree_ptr& out,
                          const std::string& base_diretory,
                          const std::string& relative_file_path,
                          const pqrs::string::replacement& replacement) const
  {
    try {
      out.reset(new boost::property_tree::ptree());

      std::string path;
      if (boost::starts_with(relative_file_path, "/")) {
        path = relative_file_path;
      } else {
        path = base_diretory + "/" + relative_file_path;
      }

      std::string xml;
      if (replacement.empty()) {
        pqrs::string::string_from_file(xml, path.c_str());
      } else {
        pqrs::string::string_by_replacing_double_curly_braces_from_file(xml, path.c_str(), replacement);
      }
      if (xml.empty()) {
        error_information_.set(path + " is not found.");
        return;
      }

      std::stringstream istream(xml, std::stringstream::in);

      int flags = boost::property_tree::xml_parser::no_comments;

      boost::property_tree::read_xml(istream, *out, flags);

    } catch (std::exception& e) {
      std::string what = e.what();
{'fraction_non_alphanumeric': 0.1282979407979408, 'fraction_numerical': 0.00016087516087516087, 'mean_word_length': 2.3252206472318804, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 146}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1548531', 'n_tokens_mistral': 3698, 'n_tokens_neox': 3621, 'n_words': 470}