<commit_before>package cc.factorie.app.nlp.mention

import scala.collection.mutable.HashSet
import scala.collection.mutable.ArrayBuffer
import cc.factorie.app.nlp._
import cc.factorie.app.nlp.parse.ParseTree
import cc.factorie.app.nlp.ner.NerSpan
import cc.factorie.app.nlp.pos.PTBPosLabel
import scala.Some

object ParseBasedMentionFinding extends DocumentAnnotator {

  private final val PERSONAL_PRONOUNS = Seq("PRP", "PRP$")
  private final val COMMON_NOUNS      = Seq("NN" , "NNS")
  private final val PROPER_NOUNS      = Seq("NNP", "NNPS")
  private final val ALL_NOUNS         = Seq("NN","NNS","NNP","NNPS","PRP","PRP$")

  private def isPersonalPronoun(t: Token) = PERSONAL_PRONOUNS.contains(t.posLabel.categoryValue.toUpperCase)
  private def isCommonNoun     (t: Token) = COMMON_NOUNS.contains(t.posLabel.categoryValue.toUpperCase)
  private def isProperNoun     (t: Token) = PROPER_NOUNS.contains(t.posLabel.categoryValue.toUpperCase)
  private def isNoun           (t: Token) = ALL_NOUNS.contains(t.posLabel.categoryValue.toUpperCase)

  private def nerSpans(doc: Document): Seq[Mention] = {
    (for (section <- doc.sections; span <- section.spansOfClass[NerSpan]) yield
      Mention(section, span.start, span.length, span.length - 1) //this sets the head token idx to be the last token in the span
      ).toSeq
  }

  // [Assumes personal pronouns are single tokens.]
  private def personalPronounSpans(doc: Document): Seq[Mention] = {
    (for (section <- doc.sections; s <- section.sentences;
           (t,i) <- s.tokens.zipWithIndex if isPersonalPronoun(t)) yield
        Mention(section, s.start + i, 1,0)
      ).toSeq
  }

  private def getHeadTokenIdx(m: Mention): Int = {
   val tokenIdxInSection =  getHead(
      m.span.head.sentence.parse,
      m.start until (m.start + m.length) //these are section-level offsets
    )
    val tokenIdxInSpan = tokenIdxInSection - m.span.start
    assert(tokenIdxInSpan >= 0 && tokenIdxInSpan <= m.span.length)
    tokenIdxInSpan
  }
  //this expects as input indices in the **document section** not the sentence
  //note that this never returns the root as the head, it always returns a pointer to an actual token in the sentence
  //it will either return the root of a parse tree span, or a token that is a child of the root
  def getHead(parse: ParseTree, subtree: Seq[Int]): Int = {
    val sentenceLevelIndices = subtree.map(i => i - parse.sentence.start)
    var curr = sentenceLevelIndices.head
    val leftBoundary = sentenceLevelIndices.head
    val rightBoundary = sentenceLevelIndices.last
    while(parse.parentIndex(curr) > 0 && containedInInterval(leftBoundary,rightBoundary,parse.parentIndex(curr))){
      curr = parse.parentIndex(curr)
    }
    curr + parse.sentence.start  //this shifts it back to have section-level indices
  }

  private def containedInInterval(left: Int, right: Int, testIndex: Int): Boolean = {
    testIndex >= left && testIndex <= right
  }

  final val copularVerbs = collection.immutable.HashSet[String]() ++ Seq("is","are","was","'m")

  private def nounPhraseSpans(doc: Document, nounFilter: Token => Boolean): Seq[Mention] = (

    for (section <- doc.sections; s <- section.sentences;
         usedTokens = new HashSet[Token]();
         (t, si) <- s.tokens.zipWithIndex if nounFilter(t)) yield {

      //These two filtering rules are from 'Mention Detection: Heuristics for the OntoNotes annotations'
      val prevTokenIsCopular = si > 0 && copularVerbs.contains(s.tokens(si-1).string.toLowerCase)
      val copularPhrase = s.parse.parentIndex(si)!= -1 && prevTokenIsCopular && s.parse.parent(si).posLabel.value == "VB"

      val parentIsNoun = s.parse.parentIndex(si) == -1 && s.parse.parent(si) != null && isNoun(s.parse.parent(si))
      val prevWordIsComma = t.hasPrev && t.prev.string == ","
      val prevPhraseIsNP = if(si > 1) usedTokens.contains(s.tokens(si - 2)) else false
      val apposition =  parentIsNoun && prevWordIsComma && prevPhraseIsNP

      // skip tokens that are:
      //   1. part of a copular phrase,
      //   2. appositive clauses,
      //   3. other nouns in a noun phrase
      if (copularPhrase || apposition || parentIsNoun) None
      else {
        val subtree = (Seq(t) ++ s.parse.subtree(si)).sortBy(_.position)
        usedTokens ++= subtree
        val (start, length) = subtree.size match {
          // leaf of the parse
          case 1 => t.position -> 1
          // non-leaf
          case _ => subtree.head.position -> (subtree.last.position - subtree.head.position + 1)
        }<commit_msg>mention: Fixing code which was working and got broken through merges<commit_after>package cc.factorie.app.nlp.mention

import scala.collection.mutable.HashSet
import scala.collection.mutable.ArrayBuffer
import cc.factorie.app.nlp._
import cc.factorie.app.nlp.parse.ParseTree
import cc.factorie.app.nlp.ner.NerSpan
import cc.factorie.app.nlp.pos.PTBPosLabel
import scala.Some

object ParseBasedMentionFinding extends DocumentAnnotator {

  private final val PERSONAL_PRONOUNS = Seq("PRP", "PRP$")
  private final val COMMON_NOUNS      = Seq("NN" , "NNS")
  private final val PROPER_NOUNS      = Seq("NNP", "NNPS")
  private final val ALL_NOUNS         = Seq("NN","NNS","NNP","NNPS","PRP","PRP$")

  private def isPersonalPronoun(t: Token) = PERSONAL_PRONOUNS.contains(t.posLabel.categoryValue.toUpperCase)
  private def isCommonNoun     (t: Token) = COMMON_NOUNS.contains(t.posLabel.categoryValue.toUpperCase)
  private def isProperNoun     (t: Token) = PROPER_NOUNS.contains(t.posLabel.categoryValue.toUpperCase)
  private def isNoun           (t: Token) = ALL_NOUNS.contains(t.posLabel.categoryValue.toUpperCase)

  private def nerSpans(doc: Document): Seq[Mention] = {
    (for (section <- doc.sections; span <- section.spansOfClass[NerSpan]) yield
      Mention(section, span.start, span.length, span.length - 1) //this sets the head token idx to be the last token in the span
      ).toSeq
  }

  // [Assumes personal pronouns are single tokens.]
  private def personalPronounSpans(doc: Document): Seq[Mention] = {
    (for (section <- doc.sections; s <- section.sentences;
           (t,i) <- s.tokens.zipWithIndex if isPersonalPronoun(t)) yield
        Mention(section, s.start + i, 1,0)
      ).toSeq
  }

  private def getHeadTokenIdx(m: Mention): Int = {
   val tokenIdxInSection =  getHead(
      m.span.head.sentence.parse,
      m.start until (m.start + m.length) //these are section-level offsets
    )
    val tokenIdxInSpan = tokenIdxInSection - m.span.start
    assert(tokenIdxInSpan >= 0 && tokenIdxInSpan <= m.span.length)
    tokenIdxInSpan
  }
  //this expects as input indices in the **document section** not the sentence
  //note that this never returns the root as the head, it always returns a pointer to an actual token in the sentence
  //it will either return the root of a parse tree span, or a token that is a child of the root
  def getHead(parse: ParseTree, subtree: Seq[Int]): Int = {
    val sentenceLevelIndices = subtree.map(i => i - parse.sentence.start)
    var curr = sentenceLevelIndices.head
    val leftBoundary = sentenceLevelIndices.head
    val rightBoundary = sentenceLevelIndices.last
    while(parse.parentIndex(curr) > 0 && containedInInterval(leftBoundary,rightBoundary,parse.parentIndex(curr))){
      curr = parse.parentIndex(curr)
    }
    curr + parse.sentence.start  //this shifts it back to have section-level indices
  }

  private def containedInInterval(left: Int, right: Int, testIndex: Int): Boolean = {
    testIndex >= left && testIndex <= right
  }

  final val copularVerbs = collection.immutable.HashSet[String]() ++ Seq("is","are","was","'m")

  private def nounPhraseSpans(doc: Document, nounFilter: Token => Boolean): Seq[Mention] = (

    for (section <- doc.sections; s <- section.sentences;
         usedTokens = new HashSet[Token]();
         (t, si) <- s.tokens.zipWithIndex if nounFilter(t)) yield {

      //These two filtering rules are from 'Mention Detection: Heuristics for the OntoNotes annotations'
      val prevTokenIsCopular = si > 0 && copularVerbs.contains(s.tokens(si-1).string.toLowerCase)
      val copularPhrase = s.parse.parentIndex(si)!= -1 && prevTokenIsCopular && s.parse.parent(si).posLabel.value == "VB"

      val parentIsNoun = s.parse.parent(si) != null && isNoun(s.parse.parent(si))
      val prevWordIsComma = t.hasPrev && t.prev.string == ","
      val prevPhraseIsNP = if(si > 1) usedTokens.contains(s.tokens(si - 2)) else false
      val apposition =  parentIsNoun && prevWordIsComma && prevPhraseIsNP

      // skip tokens that are:
      //   1. part of a copular phrase,
      //   2. appositive clauses,
      //   3. other nouns in a noun phrase
      if (copularPhrase || apposition || parentIsNoun) None
      else {
        val subtree = (Seq(t) ++ s.parse.subtree(si)).sortBy(_.position)
        usedTokens ++= subtree
        val (start, length) = subtree.size match {
          // leaf of the parse
          case 1 => t.position -> 1
          // non-leaf
          case _ => subtree.head.position -> (subtree.last.position - subtree.head.position + 1)
        }
{'fraction_non_alphanumeric': 0.11711908931698774, 'fraction_numerical': 0.0036120840630472855, 'mean_word_length': 3.334440227703985, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5629099', 'n_tokens_mistral': 3006, 'n_tokens_neox': 2912, 'n_words': 766}