<commit_before>package joshua.decoder.hypergraph;

import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.PriorityQueue;
import java.util.logging.Logger;

import joshua.corpus.Vocabulary;
import joshua.decoder.JoshuaConfiguration;
import joshua.decoder.chart_parser.ComputeNodeResult;
import joshua.decoder.ff.FeatureFunction;
import joshua.decoder.ff.FeatureVector;
import joshua.decoder.ff.tm.Rule;
import joshua.decoder.ff.tm.BilingualRule;
import joshua.util.CoIterator;
import joshua.util.Regex;
import joshua.util.io.UncheckedIOException;

/**
 * This class implements lazy k-best extraction on a hyper-graph. To seed the kbest extraction, it
 * only needs that each hyperedge should have the best_cost properly set, and it does not require
 * any list being sorted. Instead, the priority queue heap_cands will do internal sorting In fact,
 * the real crucial cost is the transition-cost at each hyperedge. We store the best-cost instead of
 * the transition cost since it is easy to do pruning and find one-best. Moreover, the transition
 * cost can be recovered by get_transition_cost(), though somewhat expensive.
 * 
 * To recover the model cost for each individual model, we should either have access to the model,
 * or store the model cost in the hyperedge. (For example, in the case of disk-hypergraph, we need
 * to store all these model cost at each hyperedge.)
 * 
 * @author <NAME>, <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 */
public class KBestExtractor {

  /** Logger for this class. */
  private static final Logger logger = Logger.getLogger(KBestExtractor.class.getName());

  private final HashMap<HGNode, VirtualNode> virtualNodesTbl = new HashMap<HGNode, VirtualNode>();

  static String rootSym = "ROOT";
  static int rootID;// TODO: bug

  // configuratoin option
  private boolean extractUniqueNbest = true;
  private boolean includeAlign = false;
  private boolean isMonolingual = false;
  private boolean performSanityCheck = true;

  private int sentID;

  private FeatureVector weights;

  public KBestExtractor(FeatureVector weights, boolean extractUniqueNbest,
      boolean includeAlign, 
      boolean isMonolingual, boolean performSanityCheck) {
    rootID = Vocabulary.id(rootSym);

    this.weights = weights;
    this.extractUniqueNbest = extractUniqueNbest;
    this.includeAlign = includeAlign;
    this.isMonolingual = isMonolingual;
    this.performSanityCheck = performSanityCheck;
    // System.out.println("===============sanitycheck="+performSanityCheck);
  }

  // k start from 1
  // ***************** you may need to reset_state() before you call this function for the first
  // time
  public String getKthHyp(HGNode node, int k, int sentID, List<FeatureFunction> models) {

    this.sentID = sentID;
    VirtualNode virtualNode = addVirtualNode(node);

    // ==== setup the kbest at each hgnode
    DerivationState derivationState = virtualNode.lazyKBestExtractOnNode(this, k);
    if (derivationState == null) {
      return null;
    } else {

      // ==== read the kbest from each hgnode and convert to output format
      FeatureVector features = new FeatureVector();

//      return derivationState.getDerivation(this, features, models, 0);
      String outputString = JoshuaConfiguration.outputFormat
          .replace("%i", Integer.toString(sentID))
          .replace("%s", derivationState.getHypothesis(this, false, features, models))
          .replace("%t", derivationState.getHypothesis(this, true, features, models))
          .replace("%f", features.toString())
          .replace("%c", String.format("%.3f", -derivationState.cost));
      
      return outputString;
    }
  }

  /*
   * public void getNumNodesAndEdges(HGNode it, int k, int[] numNodesAndEdges) { //==== setup the
   * kbest at each hgnode VirtualNode virtualNode = addVirtualNode(it); DerivationState cur =
   * virtualNode.lazyKBestExtractOnNode(Vocabulary, this, k); if( cur==null){ numNodesAndEdges[0]=0;
   * numNodesAndEdges[1]=0; }else{ cur.getNumNodesAndEdges(this, numNodesAndEdges); } }<commit_msg>bugfix: need to call resetState() before calling getHypothesis() again<commit_after>package joshua.decoder.hypergraph;

import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.PriorityQueue;
import java.util.logging.Logger;

import joshua.corpus.Vocabulary;
import joshua.decoder.JoshuaConfiguration;
import joshua.decoder.chart_parser.ComputeNodeResult;
import joshua.decoder.ff.FeatureFunction;
import joshua.decoder.ff.FeatureVector;
import joshua.decoder.ff.tm.Rule;
import joshua.util.CoIterator;
import joshua.util.io.UncheckedIOException;

/**
 * This class implements lazy k-best extraction on a hyper-graph. To seed the kbest extraction, it
 * only needs that each hyperedge should have the best_cost properly set, and it does not require
 * any list being sorted. Instead, the priority queue heap_cands will do internal sorting In fact,
 * the real crucial cost is the transition-cost at each hyperedge. We store the best-cost instead of
 * the transition cost since it is easy to do pruning and find one-best. Moreover, the transition
 * cost can be recovered by get_transition_cost(), though somewhat expensive.
 * 
 * To recover the model cost for each individual model, we should either have access to the model,
 * or store the model cost in the hyperedge. (For example, in the case of disk-hypergraph, we need
 * to store all these model cost at each hyperedge.)
 * 
 * @author <NAME>, <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 */
public class KBestExtractor {

  /** Logger for this class. */
  private static final Logger logger = Logger.getLogger(KBestExtractor.class.getName());

  private final HashMap<HGNode, VirtualNode> virtualNodesTbl = new HashMap<HGNode, VirtualNode>();

  static String rootSym = "ROOT";
  static int rootID;// TODO: bug

  // configuratoin option
  private boolean extractUniqueNbest = true;
  private boolean includeAlign = false;
  private boolean isMonolingual = false;
  private boolean performSanityCheck = true;

  private int sentID;

  private FeatureVector weights;

  public KBestExtractor(FeatureVector weights, boolean extractUniqueNbest,
      boolean includeAlign, 
      boolean isMonolingual, boolean performSanityCheck) {
    rootID = Vocabulary.id(rootSym);

    this.weights = weights;
    this.extractUniqueNbest = extractUniqueNbest;
    this.includeAlign = includeAlign;
    this.isMonolingual = isMonolingual;
    this.performSanityCheck = performSanityCheck;
    // System.out.println("===============sanitycheck="+performSanityCheck);
  }

  // k start from 1
  // ***************** you may need to reset_state() before you call this function for the first
  // time
  public String getKthHyp(HGNode node, int k, int sentID, List<FeatureFunction> models) {

    this.sentID = sentID;
    VirtualNode virtualNode = addVirtualNode(node);

    // ==== setup the kbest at each hgnode
    DerivationState derivationState = virtualNode.lazyKBestExtractOnNode(this, k);
    if (derivationState == null) {
      return null;
    } else {

      // ==== read the kbest from each hgnode and convert to output format
      FeatureVector features = new FeatureVector();

//      return derivationState.getDerivation(this, features, models, 0);

      String outputString = JoshuaConfiguration.outputFormat
          .replace("%s", derivationState.getHypothesis(this, false, features, models))
          .replace("%i", Integer.toString(sentID))
          .replace("%f", features.toString())
          .replace("%c", String.format("%.3f", -derivationState.cost));

      if (JoshuaConfiguration.outputFormat.contains("%t")) {
        resetState();
        outputString.replace("%t", derivationState.getHypothesis(this, false, features, models));
      }

      return outputString;
    }
  }

  /*
   * public void getNumNodesAndEdges(HGNode it, int k, int[] numNodesAndEdges) { //==== setup the
   * kbest at each hgnode VirtualNode virtualNode = addVirtualNode(it); DerivationState cur =
   * virtualNode.lazyKBestExtractOnNode(Vocabulary, this, k); if( cur==null){ numNodesAndEdges[0]=0;
   * numNodesAndEdges[1]=0; }else{ cur.getNumNodesAndEdges(this, numNodesAndEdges); } }
{'fraction_non_alphanumeric': 0.10304394172687434, 'fraction_numerical': 0.0016581783726163685, 'mean_word_length': 4.212345679012346, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6713611', 'n_tokens_mistral': 2470, 'n_tokens_neox': 2390, 'n_words': 751}