<commit_before>package org.sameersingh.nlp_serde.annotators

import edu.stanford.nlp.pipeline.{Annotation, StanfordCoreNLP}
import java.util.Properties
import edu.stanford.nlp.ling.CoreAnnotations._
import edu.stanford.nlp.ling.CoreLabel
import edu.stanford.nlp.util.CoreMap
import edu.stanford.nlp.trees.TreeCoreAnnotations.TreeAnnotation
import edu.stanford.nlp.semgraph.SemanticGraph
import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations.CollapsedCCProcessedDependenciesAnnotation
import edu.stanford.nlp.dcoref.CorefChain
import edu.stanford.nlp.dcoref.CorefCoreAnnotations.CorefChainAnnotation
import org.sameersingh.nlp_serde._
import scala.collection.JavaConversions._
import scala.Some

/**
 * @author sameer
 * @since 9/1/14.
 */
class StanfordAnnotator(val annotators: Seq[String] = Seq("tokenize", "ssplit", "pos", "lemma", "ner", "parse", "dcoref"))
  extends Annotator {
  // creates a StanfordCoreNLP object, with POS tagging, lemmatization, NER, parsing, and coreference resolution
  val props = new Properties()
  props.put("annotators", annotators.mkString(", "))
  val pipeline = new StanfordCoreNLP(props)

  def printCoreMap(c: CoreMap) {
    println(c.toShorterString())
    //println(c.keySet().map(k => k -> c.get(k)).mkString("\t", "\n\t", "\n"))
  }

  override def process(doc: Document): Document = {
    val text = doc.text

    // create an empty Annotation just with the given text
    val document = new Annotation(text)

    // run all Annotators on this text
    pipeline.annotate(document)
    //println("doc: " + document)
    //printCoreMap(document)

    // these are all the sentences in this document
    // a CoreMap is essentially a Map that uses class objects as keys and has values with custom types
    val sentences = document.get(classOf[SentencesAnnotation])

    for (sentence <- sentences) {
      //println("sen: " + sentence)
      //printCoreMap(sentence)
      val s = new Sentence()
      s.idx = doc.sentences.size
      s.chars = sentence.get(classOf[CharacterOffsetBeginAnnotation]).toInt -> sentence.get(classOf[CharacterOffsetEndAnnotation]).toInt
      s.text = sentence.get(classOf[TextAnnotation])
      for (token: CoreLabel <- sentence.get(classOf[TokensAnnotation])) {
        //println("tok: " + token)
        //printCoreMap(token)
        val t = new Token()
        t.idx = token.sentIndex()
        t.chars = token.beginPosition() -> token.endPosition()
        // t.text = token.get(classOf[TextAnnotation])
        t.text = token.originalText()
        t.lemma = Some(token.lemma())
        val stem = token.get(classOf[StemAnnotation])
        t.pos = Some(token.get(classOf[PartOfSpeechAnnotation]))
        t.ner = Some(token.ner())
        s.tokens += t
      }

      // this is the parse tree of the current sentence
      val tree = sentence.get(classOf[TreeAnnotation])
      s.parseTree = Some(tree.toString)
      // this is the Stanford dependency graph of the current sentence
      val dependencies = sentence.get(classOf[CollapsedCCProcessedDependenciesAnnotation])
      s.depTree = Some(dependencies.toString)

      doc.sentences += s
    }

    // This is the coreference link graph
    // Each chain stores a set of mentions that link to each other,
    // along with a method for getting the most representative mention
    // Both sentence and token offsets start at 1!
    val graph = document.get(classOf[CorefChainAnnotation])
    for ((id, chain) <- graph) {
      val e = new Entity()
      e.id = id.toInt
      val rep = chain.getRepresentativeMention
      for(mention <- chain.getMentionsInTextualOrder) {<commit_msg>cleanup stanford annotator, and fix token index<commit_after>package org.sameersingh.nlp_serde.annotators

import edu.stanford.nlp.pipeline.{Annotation, StanfordCoreNLP}
import java.util.Properties
import edu.stanford.nlp.ling.CoreAnnotations._
import edu.stanford.nlp.ling.CoreLabel
import edu.stanford.nlp.trees.TreeCoreAnnotations.TreeAnnotation
import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations.CollapsedCCProcessedDependenciesAnnotation
import edu.stanford.nlp.dcoref.CorefCoreAnnotations.CorefChainAnnotation
import org.sameersingh.nlp_serde._
import scala.collection.JavaConversions._

/**
 * @author sameer
 * @since 9/1/14.
 */
class StanfordAnnotator(val annotators: Seq[String] = Seq("tokenize", "ssplit", "pos", "lemma", "ner", "parse", "dcoref"))
  extends Annotator {
  // creates a StanfordCoreNLP object, with POS tagging, lemmatization, NER, parsing, and coreference resolution
  val props = new Properties()
  props.put("annotators", annotators.mkString(", "))
  val pipeline = new StanfordCoreNLP(props)

  override def process(doc: Document): Document = {
    val text = doc.text
    val document = new Annotation(text)
    pipeline.annotate(document)

    val sentences = document.get(classOf[SentencesAnnotation])

    for (sentence <- sentences) {
      val s = new Sentence()
      s.idx = doc.sentences.size
      s.chars = sentence.get(classOf[CharacterOffsetBeginAnnotation]).toInt -> sentence.get(classOf[CharacterOffsetEndAnnotation]).toInt
      s.text = sentence.get(classOf[TextAnnotation])
      for (token: CoreLabel <- sentence.get(classOf[TokensAnnotation])) {
        val t = new Token()
        t.idx = token.index()
        t.chars = token.beginPosition() -> token.endPosition()
        // t.text = token.get(classOf[TextAnnotation])
        t.text = token.originalText()
        t.lemma = Some(token.lemma())
        val stem = token.get(classOf[StemAnnotation])
        t.pos = Some(token.get(classOf[PartOfSpeechAnnotation]))
        t.ner = Some(token.ner())
        s.tokens += t
      }

      // this is the parse tree of the current sentence
      val tree = sentence.get(classOf[TreeAnnotation])
      s.parseTree = Some(tree.toString)
      // this is the Stanford dependency graph of the current sentence
      val dependencies = sentence.get(classOf[CollapsedCCProcessedDependenciesAnnotation])
      s.depTree = Some(dependencies.toString)

      doc.sentences += s
    }

    val graph = document.get(classOf[CorefChainAnnotation])
    for ((id, chain) <- graph) {
      val e = new Entity()
      e.id = id.toInt
      val rep = chain.getRepresentativeMention
      for(mention <- chain.getMentionsInTextualOrder) {
{'fraction_non_alphanumeric': 0.11255548509828789, 'fraction_numerical': 0.0014267596702599874, 'mean_word_length': 3.558526011560694, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6722894', 'n_tokens_mistral': 1931, 'n_tokens_neox': 1888, 'n_words': 445}