<commit_before>package epic.parser
package projections

/*
 Copyright 2012 <NAME>

 Licensed under the Apache License, Version 2.0 (the "License")
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/
import breeze.collection.mutable.TriangularArray
import breeze.config.{Configuration, CommandLineParser, Help}
import breeze.util.{Encoder, Index}
import collection.immutable.BitSet
import java.io._
import epic.trees._
import collection.mutable.ArrayBuffer
import breeze.stats.distributions.{Rand, Binomial}
import epic.parser.projections.ParserChartConstraintsFactory.PruningStatistics
import breeze.linalg._
import breeze.numerics._
import java.util
import epic.lexicon.Lexicon
import epic.constraints.{CachedChartConstraintsFactory, ChartConstraints}
import epic.util.{SafeLogging, CacheBroker}
import scala.collection.GenTraversable
import com.typesafe.scalalogging.slf4j.{LazyLogging, Logger}
import java.util.concurrent.atomic.AtomicInteger

@SerialVersionUID(1L)
class ConstraintCoreGrammarAdaptor[L, W](val grammar: BaseGrammar[L], val lexicon: Lexicon[L, W],
                                         val constraintsFactory: ChartConstraints.Factory[L, W]) extends CoreGrammar[L, W] with Serializable {
  /**
   * Returns a [[epic.parser.CoreAnchoring]] for this particular sentence.
   * @param words
   * @return
   */
  def anchor(words: IndexedSeq[W]): CoreAnchoring[L, W] = CoreAnchoring.identity[L, W](grammar, lexicon, words, constraintsFactory.constraints(words))
}

/**
 * Creates labeled span scorers for a set of trees from some parser.
 * @author dlwh
 */
@SerialVersionUID(1L)
class ParserChartConstraintsFactory[L, W](val parser: Parser[L, W],
                                          val isIntermediate: L=>Boolean,
                                          threshold: Double = math.exp(-9)) extends ChartConstraints.Factory[L, W] with Serializable with SafeLogging {
  require(threshold >= 0 && threshold <= 1, s"Threshold must be between 0 and 1, but whas $threshold")
  import parser._
  def labelIndex = grammar.labelIndex

  def overallStatistics = {
    val xp = pruned.get()
    val xpt = prunedtags.get()
    val xnp = notpruned.get()
    val xnpt = notprunedtags.get()

    s"Pruned Non-Unit Labels: ${xp * 1.0 / (xp + xnp)}; Pruned Labels ${xpt * 1.0 / {xpt + xnpt}}"
  }

  val pruned = new AtomicInteger(0)
  val notpruned = new AtomicInteger(0)
  val prunedtags = new AtomicInteger(0)
  val notprunedtags = new AtomicInteger(0)


  private val synthetics = BitSet.empty ++ (0 until grammar.labelIndex.size).filter(l => isIntermediate(labelIndex.get(l)))

  def constraints(w: IndexedSeq[W]):ChartConstraints[L] = constraints(w, GoldTagPolicy.noGoldTags[L])
  def constraints(words: IndexedSeq[W], gold: GoldTagPolicy[L]):ChartConstraints[L] = {
    val charts = parser.marginal(words)
    constraints(charts, gold)
  }

  def constraints(marg: ParseMarginal[L, W], gold: GoldTagPolicy[L]): ChartConstraints[L] = {
    logger.debug(s"Building Constraints for ${marg.words}")
    assert(marg.isMaxMarginal)
    val length = marg.length
    if(marg.logPartition.isInfinite)
      throw new NoParseException("No parse for sentence we're trying to constrain!", marg.words)
    val (botLabelScores, unaryScores) = computeScores(length, marg)

    val vit = new ViterbiDecoder[L, W].extractBestParse(marg.asInstanceOf[ChartMarginal[L, W]])

    val labelThresholds = extractLabelThresholds(length,
      grammar.labelIndex.size,
      botLabelScores, grammar.labelIndex,
      gold.isGoldBotTag)
    val topLabelThresholds = extractLabelThresholds(length,
      grammar.labelIndex.size,
      unaryScores,grammar.labelIndex,
      gold.isGoldTopTag)
    for(i <- 0 until length) {
      assert(labelThresholds(i, i+1) != null && labelThresholds(i,i+1).nonEmpty, "label thresholds" + labelThresholds(i, i+1))
      assert(topLabelThresholds(i, i+1) != null && topLabelThresholds(0,length).nonEmpty, "top label thresholds" + topLabelThresholds(i, i+1))
    }
    if(topLabelThresholds(0,length) == null || !topLabelThresholds(0,length).contains(marg.grammar.rootIndex))
      throw new NoParseException("No score at the root!", marg.words)

//    val hasMaximalProjection: BitSet = BitSet.empty ++ (0 to length).filter{ i =>
//      ((labelThresholds(i) ne null) && (topLabelThresholds(i) ne null)) && ((labelThresholds(i)|topLabelThresholds(i)) -- synthetics).nonEmpty
//    }

    //, hasMaximalProjection)<commit_msg>messed up merge.<commit_after>package epic.parser
package projections

/*
 Copyright 2012 <NAME>

 Licensed under the Apache License, Version 2.0 (the "License")
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/
import breeze.collection.mutable.TriangularArray
import breeze.config.{Configuration, CommandLineParser, Help}
import breeze.util.{Encoder, Index}
import collection.immutable.BitSet
import java.io._
import epic.trees._
import collection.mutable.ArrayBuffer
import breeze.stats.distributions.{Rand, Binomial}
import epic.parser.projections.ParserChartConstraintsFactory.PruningStatistics
import breeze.linalg._
import breeze.numerics._
import java.util
import epic.lexicon.Lexicon
import epic.constraints.{CachedChartConstraintsFactory, ChartConstraints}
import epic.util.{SafeLogging, CacheBroker}
import scala.collection.GenTraversable
import com.typesafe.scalalogging.slf4j.{LazyLogging, Logger}
import java.util.concurrent.atomic.AtomicInteger

@SerialVersionUID(1L)
class ConstraintCoreGrammarAdaptor[L, W](val grammar: BaseGrammar[L], val lexicon: Lexicon[L, W],
                                         val constraintsFactory: ChartConstraints.Factory[L, W]) extends CoreGrammar[L, W] with Serializable {
  /**
   * Returns a [[epic.parser.CoreAnchoring]] for this particular sentence.
   * @param words
   * @return
   */
  def anchor(words: IndexedSeq[W]): CoreAnchoring[L, W] = CoreAnchoring.identity[L, W](grammar, lexicon, words, constraintsFactory.constraints(words))
}

/**
 * Creates labeled span scorers for a set of trees from some parser.
 * @author dlwh
 */
@SerialVersionUID(1L)
class ParserChartConstraintsFactory[L, W](val parser: Parser[L, W],
                                          val isIntermediate: L=>Boolean,
                                          threshold: Double = math.exp(-9)) extends ChartConstraints.Factory[L, W] with Serializable with SafeLogging {
  require(threshold >= 0 && threshold <= 1, s"Threshold must be between 0 and 1, but whas $threshold")
  import parser._
  def labelIndex = grammar.labelIndex

  def overallStatistics = {
    val xp = pruned.get()
    val xpt = prunedtags.get()
    val xnp = notpruned.get()
    val xnpt = notprunedtags.get()

    s"Pruned Non-Unit Labels: ${xp * 1.0 / (xp + xnp)}; Pruned Labels ${xpt * 1.0 / {xpt + xnpt}}"
  }

  val pruned = new AtomicInteger(0)
  val notpruned = new AtomicInteger(0)
  val prunedtags = new AtomicInteger(0)
  val notprunedtags = new AtomicInteger(0)


  private val synthetics = BitSet.empty ++ (0 until grammar.labelIndex.size).filter(l => isIntermediate(labelIndex.get(l)))

  def constraints(w: IndexedSeq[W]):ChartConstraints[L] = constraints(w, GoldTagPolicy.noGoldTags[L])
  def constraints(words: IndexedSeq[W], gold: GoldTagPolicy[L]):ChartConstraints[L] = {
    val charts = parser.marginal(words)
    constraints(charts, gold)
  }

  def constraints(marg: ParseMarginal[L, W], gold: GoldTagPolicy[L]): ChartConstraints[L] = {
    logger.debug(s"Building Constraints for ${marg.words}")
    assert(marg.isMaxMarginal)
    val length = marg.length
    if(marg.logPartition.isInfinite)
      throw new NoParseException("No parse for sentence we're trying to constrain!", marg.words)
    val (botLabelScores, unaryScores) = computeScores(length, marg)

    val vit = new ViterbiDecoder[L, W].extractBestParse(marg)

    val labelThresholds = extractLabelThresholds(length,
      grammar.labelIndex.size,
      botLabelScores, grammar.labelIndex,
      gold.isGoldBotTag)
    val topLabelThresholds = extractLabelThresholds(length,
      grammar.labelIndex.size,
      unaryScores,grammar.labelIndex,
      gold.isGoldTopTag)
    for(i <- 0 until length) {
      assert(labelThresholds(i, i+1) != null && labelThresholds(i,i+1).nonEmpty, "label thresholds" + labelThresholds(i, i+1))
      assert(topLabelThresholds(i, i+1) != null && topLabelThresholds(0,length).nonEmpty, "top label thresholds" + topLabelThresholds(i, i+1))
    }
    if(topLabelThresholds(0,length) == null || !topLabelThresholds(0,length).contains(marg.grammar.rootIndex))
      throw new NoParseException("No score at the root!", marg.words)

//    val hasMaximalProjection: BitSet = BitSet.empty ++ (0 to length).filter{ i =>
//      ((labelThresholds(i) ne null) && (topLabelThresholds(i) ne null)) && ((labelThresholds(i)|topLabelThresholds(i)) -- synthetics).nonEmpty
//    }

    //, hasMaximalProjection)
{'fraction_non_alphanumeric': 0.10884143196120911, 'fraction_numerical': 0.00722170638605179, 'mean_word_length': 4.195069667738478, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6815886', 'n_tokens_mistral': 3053, 'n_tokens_neox': 2879, 'n_words': 772}