<commit_before>package cc.factorie.app.nlp.mention

import cc.factorie.app.nlp._
import cc.factorie.app.nlp.pos.PennPosLabel
import cc.factorie.app.nlp.ner.{NerSpan, NerLabel}
import scala.collection.mutable.ArrayBuffer
import cc.factorie.variable.Span

/**
 * User: apassos
 * Date: 8/6/13
 * Time: 3:34 PM
 */

class NerMentionList extends MentionList

object NerAndPronounMentionFinder extends DocumentAnnotator {
  def prereqAttrs = Seq(classOf[NerLabel], classOf[PennPosLabel])
  def postAttrs = Seq(classOf[NerMentionList], classOf[MentionEntityType])
  override def tokenAnnotationString(token:Token): String = token.document.attr[MentionList].filter(mention => mention.contains(token)) match { case ms:Seq[Mention] if ms.length > 0 => ms.map(m => m.attr[MentionType].categoryValue+":"+ m.attr[MentionEntityType].categoryValue +":" +m.indexOf(token)).mkString(","); case _ => "_" }

  val upperCase = "[A-Z]+".r
  def getNerSpans(doc: Document): Seq[(String,TokenSpan)] = {
    val spans = ArrayBuffer[(String,TokenSpan)]()
    for (s <- doc.sections;t <- s.tokens) {
      if (t.attr[NerLabel].categoryValue != "O") {
        val attr = t.attr[NerLabel].categoryValue.split("-")
        if (attr(0) == "U") {
          val lab = attr(1)
          spans += (lab -> new TokenSpan(s, t.positionInSection, 1))
        } else if (attr(0) == "B") {
          val lab = attr(1)
          if(t.hasNext) {
            var lookFor = t.next
            while (lookFor.hasNext && lookFor.attr[NerLabel].categoryValue.matches("(I|L)-" + attr(1))) lookFor = lookFor.next
            spans += (lab -> new TokenSpan(s, t.positionInSection, lookFor.positionInSection - t.positionInSection))
          } else {
            spans += (lab -> new TokenSpan(s, t.positionInSection, 1))
          }
        }
      } else {
        if ( t.string.length > 2 && !t.containsLowerCase && upperCase.findFirstIn(t.string).nonEmpty && (t.getNext ++ t.getPrev).exists(i => i.containsLowerCase)) {
          spans += ("ORG" -> new TokenSpan(s, t.positionInSection, 1))
        } else if (t.posLabel.categoryValue == "NNP") {
          spans += ("PER" -> new TokenSpan(s, t.positionInSection, 1))
        }
      }<commit_msg>app.nlp.mention: Fixing NerAndPronounMentionFinder to call Thursday a MISC instead of a PER<commit_after>package cc.factorie.app.nlp.mention

import cc.factorie.app.nlp._
import cc.factorie.app.nlp.pos.PennPosLabel
import cc.factorie.app.nlp.ner.{NerSpan, NerLabel}
import scala.collection.mutable.ArrayBuffer
import cc.factorie.variable.Span

/**
 * User: apassos
 * Date: 8/6/13
 * Time: 3:34 PM
 */

class NerMentionList extends MentionList

object NerAndPronounMentionFinder extends DocumentAnnotator {
  def prereqAttrs = Seq(classOf[NerLabel], classOf[PennPosLabel])
  def postAttrs = Seq(classOf[NerMentionList], classOf[MentionEntityType])
  override def tokenAnnotationString(token:Token): String = token.document.attr[MentionList].filter(mention => mention.contains(token)) match { case ms:Seq[Mention] if ms.length > 0 => ms.map(m => m.attr[MentionType].categoryValue+":"+ m.attr[MentionEntityType].categoryValue +":" +m.indexOf(token)).mkString(","); case _ => "_" }

  val upperCase = "[A-Z]+".r
  def getNerSpans(doc: Document): Seq[(String,TokenSpan)] = {
    val spans = ArrayBuffer[(String,TokenSpan)]()
    for (s <- doc.sections;t <- s.tokens) {
      if (t.attr[NerLabel].categoryValue != "O") {
        val attr = t.attr[NerLabel].categoryValue.split("-")
        if (attr(0) == "U") {
          val lab = attr(1)
          spans += (lab -> new TokenSpan(s, t.positionInSection, 1))
        } else if (attr(0) == "B") {
          val lab = attr(1)
          if(t.hasNext) {
            var lookFor = t.next
            while (lookFor.hasNext && lookFor.attr[NerLabel].categoryValue.matches("(I|L)-" + attr(1))) lookFor = lookFor.next
            spans += (lab -> new TokenSpan(s, t.positionInSection, lookFor.positionInSection - t.positionInSection))
          } else {
            spans += (lab -> new TokenSpan(s, t.positionInSection, 1))
          }
        }
      } else {
        if ( t.string.length > 2 && !t.containsLowerCase && upperCase.findFirstIn(t.string).nonEmpty && (t.getNext ++ t.getPrev).exists(i => i.containsLowerCase)) {
          spans += ("ORG" -> new TokenSpan(s, t.positionInSection, 1))
        } else if (t.posLabel.categoryValue == "NNP") {
          spans += ("MISC" -> new TokenSpan(s, t.positionInSection, 1))
        }
      }
{'fraction_non_alphanumeric': 0.15599196607900023, 'fraction_numerical': 0.008033920999776836, 'mean_word_length': 3.212406015037594, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4629589', 'n_tokens_mistral': 1540, 'n_tokens_neox': 1519, 'n_words': 293}