<commit_before>#include "extractor.h"

#include <cassert>
#include <map>
#include <vector>

#include "taco/base/exception.h"

#include "tools-common/relation/relation.h"

namespace taco {
namespace tool {
namespace m3 {

void Extractor::Extract(Tree &t, std::set<Relation> &relations,
                        std::vector<std::string> &warnings) {
  relation_vec_.clear();
  node_to_relation_.clear();

  // Visit nodes extracting noun-modifier agreement relations.
  for (std::vector<Tree *>::const_iterator p = t.children().begin();
       p != t.children().end(); ++p) {
    Extract(**p, warnings);
  }

  // Add relations to output set.
  for (RelationVec::const_iterator p = relation_vec_.begin();
       p != relation_vec_.end(); ++p) {
    relations.insert(**p);
  }

  // Mop up any unclaimed modifiers and nouns to produce single-word
  // selector-target sets.  Whilst those selectors / targets don't participate
  // in an agreement relation in this particular parse tree, the rules they
  // end up in may become part of a derivation where there is such a relation.
  std::vector<Tree *> leaves;
  t.GetLeaves(leaves);
  for (std::vector<Tree *>::const_iterator p = leaves.begin();
       p != leaves.end(); ++p) {
    Tree *leaf = *p;
    // Already in a relation?
    if (node_to_relation_.find(leaf) != node_to_relation_.end()) {
      continue;
    }
    Tree *preterminal = leaf->parent();
    if (IsNoCm(*preterminal) || IsModifier(*preterminal)) {
      Relation r;
      AddNodeToRelation(*preterminal, r);
      AddNodeToRelation(*leaf, r);
      relations.insert(r);
    }
  }
}

void Extractor::Extract(Tree &t, std::vector<std::string> &warnings) {
  // Recursively visit children first.
  for (std::vector<Tree *>::const_iterator p = t.children().begin();
       p != t.children().end(); ++p) {
    Extract(**p, warnings);
  }

  // Check if this is a common noun.
  if (!IsNoCm(t)) {
    return;
  }

  // Look for the closest sibling to the left.
  Tree *left_sibling = ClosestLeftSibling(t);
  if (!left_sibling) {
    return;
  }

  // If the sibling is also a common noun then ignore this noun.  Any modifiers
  // to the left will agree with that noun, not this one.
  if (IsNoCm(*left_sibling)) {
    return;
  }

  // Create a new relation and add the noun.
  boost::shared_ptr<Relation> r(new Relation());
  relation_vec_.push_back(r);
  AddNodeToRelation(t, *r);
  AddNodeToRelation(*(t.GetChild(0)), *r);

  // Add any Det / Aj siblings to the relation, terminating after the first Det.
  while (true) {
    if (IsDet(*left_sibling)) {
      AddUnaryChainToRelation(*left_sibling, *r);
      break;
    }
    if (DominatesAj(*left_sibling)) {
      AddAjSubtreeToRelation(*left_sibling, *r);
    }
    left_sibling = ClosestLeftSibling(*left_sibling);
    if (!left_sibling) {
      break;
    }
  }
}<commit_msg>Fix bug in previous commit.<commit_after>#include "extractor.h"

#include <cassert>
#include <map>
#include <vector>

#include "taco/base/exception.h"

#include "tools-common/relation/relation.h"

namespace taco {
namespace tool {
namespace m3 {

void Extractor::Extract(Tree &t, std::set<Relation> &relations,
                        std::vector<std::string> &warnings) {
  relation_vec_.clear();
  node_to_relation_.clear();

  // Visit nodes extracting noun-modifier agreement relations.
  for (std::vector<Tree *>::const_iterator p = t.children().begin();
       p != t.children().end(); ++p) {
    Extract(**p, warnings);
  }

  // Add relations to output set.
  for (RelationVec::const_iterator p = relation_vec_.begin();
       p != relation_vec_.end(); ++p) {
    relations.insert(**p);
  }

  // Mop up any unclaimed modifiers and nouns to produce single-word
  // selector-target sets.  Whilst those selectors / targets don't participate
  // in an agreement relation in this particular parse tree, the rules they
  // end up in may become part of a derivation where there is such a relation.
  std::vector<Tree *> leaves;
  t.GetLeaves(leaves);
  for (std::vector<Tree *>::const_iterator p = leaves.begin();
       p != leaves.end(); ++p) {
    Tree *leaf = *p;
    // Already in a relation?
    if (node_to_relation_.find(leaf) != node_to_relation_.end()) {
      continue;
    }
    Tree *preterminal = leaf->parent();
    if (IsNoCm(*preterminal) || IsModifier(*preterminal)) {
      Relation r;
      AddNodeToRelation(*preterminal, r);
      AddNodeToRelation(*leaf, r);
      relations.insert(r);
    }
  }
}

void Extractor::Extract(Tree &t, std::vector<std::string> &warnings) {
  // Recursively visit children first.
  for (std::vector<Tree *>::const_iterator p = t.children().begin();
       p != t.children().end(); ++p) {
    Extract(**p, warnings);
  }

  // Check if this is a common noun.
  if (!IsNoCm(t)) {
    return;
  }

  // Look for the closest sibling to the left.
  Tree *left_sibling = ClosestLeftSibling(t);
  if (!left_sibling) {
    return;
  }

  // If the sibling is also a common noun then ignore this noun.  Any modifiers
  // to the left will agree with that noun, not this one.
  if (IsNoCm(*left_sibling)) {
    return;
  }

  // Create a new relation.
  boost::shared_ptr<Relation> r(new Relation());
  relation_vec_.push_back(r);

  // Add the noun.
  AddNodeToRelation(t, *r);
  AddNodeToRelation(*(t.GetChild(0)), *r);

  // Add the parent node.
  AddNodeToRelation(*(t.parent()), *r);

  // Add any Det / Aj siblings to the relation, terminating after the first Det.
  while (true) {
    if (IsDet(*left_sibling)) {
      AddUnaryChainToRelation(*left_sibling, *r);
      break;
    }
    if (DominatesAj(*left_sibling)) {
      AddAjSubtreeToRelation(*left_sibling, *r);
    }
    left_sibling = ClosestLeftSibling(*left_sibling);
    if (!left_sibling) {
      break;
    }
  }
}
{'fraction_non_alphanumeric': 0.14325259515570934, 'fraction_numerical': 0.0006920415224913495, 'mean_word_length': 3.2444933920704844, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5983651', 'n_tokens_mistral': 1972, 'n_tokens_neox': 1932, 'n_words': 513}