<commit_before>#include "hext/match-tree.h"
#include "hext/rule.h"


namespace hext {


match_tree::match_tree()
: children(),
  matches(),
  r(nullptr)
{
}

match_tree * match_tree::append_child_and_own(std::unique_ptr<match_tree> m)
{
  this->children.push_back(std::move(m));
  return this->children.back().get();
}

void match_tree::append_match(const name_value_pair& p)
{
  this->matches.push_back(p);
}

void match_tree::json_print(std::ostream& out) const
{
  for(const auto& c : this->children)
  {
    out << "{";
    infix_ostream_iterator<std::string> it_out(out, ", ");
    this->json_print_recursive(it_out);
    out << "}\n";
  }
}

void
match_tree::json_print_recursive(
  infix_ostream_iterator<std::string>& out
) const
{
  for(const auto& p : this->matches)
  {
    std::string str("\"");
    str.append(util::escape_quotes(p.first))
       .append("\": \"")
       .append(util::escape_quotes(p.second))
       .append("\"");
    out = str;
  }

  for(const auto& c : this->children)
    c->json_print_recursive(out);
}

void match_tree::print(std::ostream& out) const
{
  out << "digraph match_tree {\n"
      << "    node [fontname=\"Arial\"];\n";
  this->print_dot(out);
  out << "}\n";
}

void match_tree::print_dot(std::ostream& out, int parent_id) const
{
  static int node_index = 0;
  int this_node = ++node_index;

  std::string label;
  if( this->r == nullptr || this->r->get_tag_name().empty() )
    label.append("[rule]");
  else
    label.append(this->r->get_tag_name());

  for(const auto& m : this->matches)
  {
    label.append(" ");
    label.append(m.first);
  }

  if( this->r && this->r->children_size() == 0 )
    label.append("*");

  out << "    node_" << this_node << " [label=\"" << label << "\"];\n";
  if( parent_id )
    out << "    node_" << parent_id << " -> node_" << this_node << ";\n";

  for(const auto& c : this->children)
    c->print_dot(out, this_node);
}

void match_tree::set_rule(const rule * matching_rule)
{
  this->r = matching_rule;
}

void match_tree::filter()
{
  bool filter_ret = this->filter_recursive();

}

bool match_tree::filter_recursive()
{
  // TODO: move this into matcher?
  if( this->children.size() == 1 && this->matches.empty() )
  {
    // TODO: is this bullet-proof?
    std::swap(this->children, this->children.front()->children);
  }

  // if the matching rule has no more children, we have a complete
  // match of a rule path, therefore we want to keep this branch by
  // returning false.
  if( this->r && this->r->children_size() == 0 )
    return false;

  for(auto& c : this->children)
  {
    if( c->filter_recursive() )
      c.reset(nullptr);
  }

  // erase all empty unique_ptr
  this->children.erase(
    std::remove(this->children.begin(), this->children.end(), nullptr),
    this->children.end()
  );

  // if there is no matching rule and there are no children, the branch
  // is non matching and must be removed
  if( this->r == nullptr )
    return this->children.empty();
  // there must be at least as many matches as there are child-rules,
  // if not, this branch is non matching and must be removed
  else
    return this->r->children_size() > this->children.size();<commit_msg>enable koenig lookup when swapping, also swap member r<commit_after>#include "hext/match-tree.h"
#include "hext/rule.h"


namespace hext {


match_tree::match_tree()
: children(),
  matches(),
  r(nullptr)
{
}

match_tree * match_tree::append_child_and_own(std::unique_ptr<match_tree> m)
{
  this->children.push_back(std::move(m));
  return this->children.back().get();
}

void match_tree::append_match(const name_value_pair& p)
{
  this->matches.push_back(p);
}

void match_tree::json_print(std::ostream& out) const
{
  for(const auto& c : this->children)
  {
    out << "{";
    infix_ostream_iterator<std::string> it_out(out, ", ");
    this->json_print_recursive(it_out);
    out << "}\n";
  }
}

void
match_tree::json_print_recursive(
  infix_ostream_iterator<std::string>& out
) const
{
  for(const auto& p : this->matches)
  {
    std::string str("\"");
    str.append(util::escape_quotes(p.first))
       .append("\": \"")
       .append(util::escape_quotes(p.second))
       .append("\"");
    out = str;
  }

  for(const auto& c : this->children)
    c->json_print_recursive(out);
}

void match_tree::print(std::ostream& out) const
{
  out << "digraph match_tree {\n"
      << "    node [fontname=\"Arial\"];\n";
  this->print_dot(out);
  out << "}\n";
}

void match_tree::print_dot(std::ostream& out, int parent_id) const
{
  static int node_index = 0;
  int this_node = ++node_index;

  std::string label;
  if( this->r == nullptr || this->r->get_tag_name().empty() )
    label.append("[rule]");
  else
    label.append(this->r->get_tag_name());

  for(const auto& m : this->matches)
  {
    label.append(" ");
    label.append(m.first);
  }

  if( this->r && this->r->children_size() == 0 )
    label.append("*");

  out << "    node_" << this_node << " [label=\"" << label << "\"];\n";
  if( parent_id )
    out << "    node_" << parent_id << " -> node_" << this_node << ";\n";

  for(const auto& c : this->children)
    c->print_dot(out, this_node);
}

void match_tree::set_rule(const rule * matching_rule)
{
  this->r = matching_rule;
}

void match_tree::filter()
{
  bool filter_ret = this->filter_recursive();

}

bool match_tree::filter_recursive()
{
  if( this->children.size() == 1 && this->matches.empty() )
  {
    // enable koenig lookup
    using namespace std;
    // TODO: is this bullet-proof?
    swap(this->children, this->children.front()->children);
    swap(this->r, this->children.front()->r);
  }

  // if the matching rule has no more children, we have a complete
  // match of a rule path, therefore we want to keep this branch by
  // returning false.
  if( this->r && this->r->children_size() == 0 )
    return false;

  for(auto& c : this->children)
  {
    if( c->filter_recursive() )
      c.reset(nullptr);
  }

  // erase all empty unique_ptr
  this->children.erase(
    std::remove(this->children.begin(), this->children.end(), nullptr),
    this->children.end()
  );

  // if there is no matching rule and there are no children, the branch
  // is non matching and must be removed
  if( this->r == nullptr )
    return this->children.empty();
  // there must be at least as many matches as there are child-rules,
  // if not, this branch is non matching and must be removed
  // TODO: can this break?
  else
    return this->r->children_size() > this->children.size();
{'fraction_non_alphanumeric': 0.1773573296415936, 'fraction_numerical': 0.0012305799107829565, 'mean_word_length': 3.4201223657375937, 'pattern_counts': {'":': 2, '<': 69, '<?xml version=': 0, '>': 98, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1038346', 'n_tokens_mistral': 2315, 'n_tokens_neox': 2266, 'n_words': 516}