<commit_before>package enju.ccg.parser

import enju.ccg.lexicon.{PoS, Word, Category, TaggedSentence, TrainSentence, TestSentence, Derivation, CandAssignedSentence}
import enju.ccg.ml.{Perceptron, FeatureIndexer}
import scala.collection.mutable.{ArrayBuffer, HashMap}

class BeamSearchDecoder(val indexer:FeatureIndexer[LF],
                        val extractors:FeatureExtractors,
                        val classifier:Perceptron[ActionLabel],
                        val oracleGen:OracleGenerator,
                        override val rule:Rule,
                        val beamSize:Int,
                        val initialState:State) extends TransitionBasedParser {

  case class Candidate(path:StatePath, wrappedAction:WrappedAction, score:Double)

  case class Beam(kbest:List[StatePath]) {
    def isEmpty:Boolean = kbest.isEmpty

    def reset(candidates:List[Candidate]) = resetQuick(candidates.sortWith(_.score > _.score))
    def resetQuick(sortedCandidates:List[Candidate]) = {
      val newKBest = sortedCandidates.take(beamSize).map {
        case Candidate(path, wrappedAction, score) =>
          val newState = path.state.proceed(wrappedAction.v, wrappedAction.isGold)
          StatePath(newState, wrappedAction :: path.actionPath, score)
      }
      Beam(newKBest)
    }
    def existsGold = kbest.exists(_.state.isGold)

    def collectCandidatesTrain(sentence:TrainSentence, oracle:Oracle) = kbest.flatMap { path =>
      // currently (in deterministic-oracle), oracle actions are only defined to the gold state
      val goldActions:Seq[Action] = if (path.state.isGold) oracle.goldActions(path.state) else Nil
      // partial features (without label)
      val unlabeledFeatures = extractors.extractUnlabeledFeatures(sentence, path.state)

      possibleActions(path.state, sentence).map { action =>
        val isGold = goldActions.contains(action) // support non-deterministic oracle; currently, goldActions only contain one element so this operation is simple equality check
        val featureIdxs = unlabeledFeatures.map { _.assignLabel(action.toLabel) }.map { indexer.getIndex(_) }.toArray
        val sumScore = path.score + classifier.featureScore(featureIdxs)
        Candidate(path, WrappedAction(action, isGold, featureIdxs), sumScore)
      }
    }
    def collectCandidatesTest(sentence:CandAssignedSentence) = kbest.flatMap { path =>
      val unlabeledFeatures = extractors.extractUnlabeledFeatures(sentence, path.state)
      possibleActions(path.state, sentence).map { action =>
        val featureIdxs = unlabeledFeatures.map { _.assignLabel(action.toLabel) }.map { indexer.getOrElse(_, -1) }.toArray
        val sumScore = path.score + classifier.featureScore(featureIdxs)
        Candidate(path, WrappedAction(action, false), sumScore) // do not preserve (partial) features at test time
      }
    }
  }
  object Beam {
    def init(initState:State): Beam = new Beam(StatePath(initState, Nil) :: Nil)
  }
  case class TrainingInstance(predictedPath:Option[StatePath], goldPath:Option[StatePath])

  def trainSentences(sentences: Array[TrainSentence], golds:Array[Derivation], numIters:Int):Unit = {
    (0 until numIters).foreach { i =>
      var correct = 0
      sentences.zip(golds).zipWithIndex.foreach {
        case ((sentence, derivation), numProcessed) =>
          if (trainSentence(sentence, derivation)) correct += 1
      }
      println("accuracy (" + i + "): " + correct.toDouble / sentences.size.toDouble + " [" + correct + "]")
      println("# features: " + indexer.size)
    }
  }
  def trainSentence(sentence: TrainSentence, gold:Derivation): Boolean =
    getTrainingInstance(sentence, gold) match {
      case TrainingInstance(Some(pred), Some(gold)) =>
        classifier.update(pred.fullFeatures, gold.fullFeatures)
        return pred.state.isGold
      case _ => sys.error("")<commit_msg>Don't update parameters if predict = gold at parser training<commit_after>package enju.ccg.parser

import enju.ccg.lexicon.{PoS, Word, Category, TaggedSentence, TrainSentence, TestSentence, Derivation, CandAssignedSentence}
import enju.ccg.ml.{Perceptron, FeatureIndexer}
import scala.collection.mutable.{ArrayBuffer, HashMap}

class BeamSearchDecoder(val indexer:FeatureIndexer[LF],
                        val extractors:FeatureExtractors,
                        val classifier:Perceptron[ActionLabel],
                        val oracleGen:OracleGenerator,
                        override val rule:Rule,
                        val beamSize:Int,
                        val initialState:State) extends TransitionBasedParser {

  case class Candidate(path:StatePath, wrappedAction:WrappedAction, score:Double)

  case class Beam(kbest:List[StatePath]) {
    def isEmpty:Boolean = kbest.isEmpty

    def reset(candidates:List[Candidate]) = resetQuick(candidates.sortWith(_.score > _.score))
    def resetQuick(sortedCandidates:List[Candidate]) = {
      val newKBest = sortedCandidates.take(beamSize).map {
        case Candidate(path, wrappedAction, score) =>
          val newState = path.state.proceed(wrappedAction.v, wrappedAction.isGold)
          StatePath(newState, wrappedAction :: path.actionPath, score)
      }
      Beam(newKBest)
    }
    def existsGold = kbest.exists(_.state.isGold)

    def collectCandidatesTrain(sentence:TrainSentence, oracle:Oracle) = kbest.flatMap { path =>
      // currently (in deterministic-oracle), oracle actions are only defined to the gold state
      val goldActions:Seq[Action] = if (path.state.isGold) oracle.goldActions(path.state) else Nil
      // partial features (without label)
      val unlabeledFeatures = extractors.extractUnlabeledFeatures(sentence, path.state)

      possibleActions(path.state, sentence).map { action =>
        val isGold = goldActions.contains(action) // support non-deterministic oracle; currently, goldActions only contain one element so this operation is simple equality check
        val featureIdxs = unlabeledFeatures.map { _.assignLabel(action.toLabel) }.map { indexer.getIndex(_) }.toArray
        val sumScore = path.score + classifier.featureScore(featureIdxs)
        Candidate(path, WrappedAction(action, isGold, featureIdxs), sumScore)
      }
    }
    def collectCandidatesTest(sentence:CandAssignedSentence) = kbest.flatMap { path =>
      val unlabeledFeatures = extractors.extractUnlabeledFeatures(sentence, path.state)
      possibleActions(path.state, sentence).map { action =>
        val featureIdxs = unlabeledFeatures.map { _.assignLabel(action.toLabel) }.map { indexer.getOrElse(_, -1) }.toArray
        val sumScore = path.score + classifier.featureScore(featureIdxs)
        Candidate(path, WrappedAction(action, false), sumScore) // do not preserve (partial) features at test time
      }
    }
  }
  object Beam {
    def init(initState:State): Beam = new Beam(StatePath(initState, Nil) :: Nil)
  }
  case class TrainingInstance(predictedPath:Option[StatePath], goldPath:Option[StatePath])

  def trainSentences(sentences: Array[TrainSentence], golds:Array[Derivation], numIters:Int):Unit = {
    (0 until numIters).foreach { i =>
      var correct = 0
      sentences.zip(golds).zipWithIndex.foreach {
        case ((sentence, derivation), numProcessed) =>
          if (trainSentence(sentence, derivation)) correct += 1
      }
      println("accuracy (" + i + "): " + correct.toDouble / sentences.size.toDouble + " [" + correct + "]")
      println("# features: " + indexer.size)
    }
  }
  def trainSentence(sentence: TrainSentence, gold:Derivation): Boolean =
    getTrainingInstance(sentence, gold) match {
      case TrainingInstance(Some(pred), Some(gold)) =>
        if (!pred.state.isGold) classifier.update(pred.fullFeatures, gold.fullFeatures)
        return pred.state.isGold
      case _ => sys.error("")
{'fraction_non_alphanumeric': 0.10458103426151674, 'fraction_numerical': 0.0010265622994995508, 'mean_word_length': 3.5261324041811846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5522989', 'n_tokens_mistral': 2295, 'n_tokens_neox': 2174, 'n_words': 481}