<commit_before>#include <algorithm>
#include <vector>
#include <queue>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <boost/shared_ptr.hpp>
#include <boost/functional/hash.hpp>
#include "fast_lexical_cast.hpp"
#include "tree_fragment.h"
#include "translator.h"
#include "hg.h"
#include "sentence_metadata.h"
#include "filelib.h"
#include "stringlib.h"
#include "tdict.h"
#include "verbose.h"

using namespace std;

struct Tree2StringGrammarNode {
  map<unsigned, Tree2StringGrammarNode> next;
  vector<TRulePtr> rules;
};

// this needs to be rewritten so it is fast and checks errors well
// use a lexer probably
static void ReadTree2StringGrammar(istream* in, Tree2StringGrammarNode* root, bool has_multiple_states) {
  string line;
  int lc = 0;
  while(getline(*in, line)) {
    ++lc;
    if (line.size() == 0 || line[0] == '#') continue;
    std::vector<StringPiece> fields = TokenizeMultisep(line, " ||| ");
    if (has_multiple_states && fields.size() != 4) {
      cerr << "Expected 4 fields in rule file but line " << lc << " is:\n" << line << endl;
      abort();
    }
    if (!has_multiple_states && fields.size() != 3) {
      cerr << "Expected 3 fields in rule file but line " << lc << " is:\n" << line << endl;
      abort();
    }
    
    cdec::TreeFragment rule_src(fields[has_multiple_states ? 1 : 0], true);
    // TODO transducer_state should be read from input
    const unsigned transducer_state = 0;
    Tree2StringGrammarNode* cur = &root->next[transducer_state];
    ostringstream os;
    int lhs = -(rule_src.root & cdec::ALL_MASK);
    // build source RHS for SCFG projection
    vector<int> frhs;
    // we traverse the rule_src in left to right, DFS order
    for (auto sym : rule_src) {
      //cerr << TD::Convert(sym & cdec::ALL_MASK) << endl;
      cur = &cur->next[sym];
      if (cdec::IsFrontier(sym)) {  // frontier symbols -> variables
        int nt = (sym & cdec::ALL_MASK);
        frhs.push_back(-nt);
      } else if (cdec::IsTerminal(sym)) {
        frhs.push_back(sym);
      } // else internal NT, nothing to do
    }
    os << '[' << TD::Convert(-lhs) << "] |||";
    for (auto x : frhs) {
      os << ' ';
      if (x < 0)
        os << '[' << TD::Convert(-x) << ']';
      else
        os << TD::Convert(x);
    }
    TRulePtr rule;
    if (has_multiple_states) {
      cerr << "Not implemented...\n"; abort(); // TODO read in states
    } else {
      os << " ||| " << fields[1] << " ||| " << fields[2];
      rule.reset(new TRule(os.str()));<commit_msg>deal with alignments in tree to string grammars<commit_after>#include <algorithm>
#include <vector>
#include <queue>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <boost/shared_ptr.hpp>
#include <boost/functional/hash.hpp>
#include "fast_lexical_cast.hpp"
#include "tree_fragment.h"
#include "translator.h"
#include "hg.h"
#include "sentence_metadata.h"
#include "filelib.h"
#include "stringlib.h"
#include "tdict.h"
#include "verbose.h"

using namespace std;

struct Tree2StringGrammarNode {
  map<unsigned, Tree2StringGrammarNode> next;
  vector<TRulePtr> rules;
};

// this needs to be rewritten so it is fast and checks errors well
// use a lexer probably
static void ReadTree2StringGrammar(istream* in, Tree2StringGrammarNode* root, bool has_multiple_states) {
  string line;
  int lc = 0;
  while(getline(*in, line)) {
    ++lc;
    if (line.size() == 0 || line[0] == '#') continue;
    std::vector<StringPiece> fields = TokenizeMultisep(line, " ||| ");
    if (has_multiple_states && fields.size() < 5) {
      cerr << "Expected at least 4 fields in rule file but line " << lc << " is:\n" << line << endl;
      abort();
    }
    if (!has_multiple_states && fields.size() < 4) {
      cerr << "Expected at least 3 fields in rule file but line " << lc << " is:\n" << line << endl;
      abort();
    }
    
    cdec::TreeFragment rule_src(fields[has_multiple_states ? 1 : 0], true);
    // TODO transducer_state should be read from input
    const unsigned transducer_state = 0;
    Tree2StringGrammarNode* cur = &root->next[transducer_state];
    ostringstream os;
    int lhs = -(rule_src.root & cdec::ALL_MASK);
    // build source RHS for SCFG projection
    vector<int> frhs;
    // we traverse the rule_src in left to right, DFS order
    for (auto sym : rule_src) {
      //cerr << TD::Convert(sym & cdec::ALL_MASK) << endl;
      cur = &cur->next[sym];
      if (cdec::IsFrontier(sym)) {  // frontier symbols -> variables
        int nt = (sym & cdec::ALL_MASK);
        frhs.push_back(-nt);
      } else if (cdec::IsTerminal(sym)) {
        frhs.push_back(sym);
      } // else internal NT, nothing to do
    }
    os << '[' << TD::Convert(-lhs) << "] |||";
    for (auto x : frhs) {
      os << ' ';
      if (x < 0)
        os << '[' << TD::Convert(-x) << ']';
      else
        os << TD::Convert(x);
    }
    TRulePtr rule;
    if (has_multiple_states) {
      cerr << "Not implemented...\n"; abort(); // TODO read in states
    } else {
      os << " ||| " << fields[1] << " ||| " << fields[2];
      if (fields.size() > 3) os << " ||| " << fields[3];
      rule.reset(new TRule(os.str()));
{'fraction_non_alphanumeric': 0.16276821960177845, 'fraction_numerical': 0.007345834138797603, 'mean_word_length': 3.074015748031496, 'pattern_counts': {'":': 0, '<': 135, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1535985', 'n_tokens_mistral': 1821, 'n_tokens_neox': 1771, 'n_words': 469}