<commit_before>package edu.arizona.sista.processors.corenlp

import edu.arizona.sista.discourse.rstparser.RSTParser
import edu.arizona.sista.processors._
import edu.arizona.sista.processors.shallownlp.ShallowNLPProcessor
import edu.arizona.sista.struct._
import edu.stanford.nlp.parser.lexparser.LexicalizedParser
import edu.stanford.nlp.parser.common.{ParserAnnotations, ParserUtils}
import edu.stanford.nlp.pipeline.{ParserAnnotatorUtils, StanfordCoreNLP}
import java.util.Properties
import collection.mutable.ListBuffer
import edu.stanford.nlp.ling.CoreAnnotations._
import scala.collection.JavaConversions._
import edu.stanford.nlp.util.CoreMap
import edu.stanford.nlp.ling.CoreAnnotations
import edu.stanford.nlp.dcoref.CorefCoreAnnotations.CorefChainAnnotation
import edu.stanford.nlp.trees.{GrammaticalStructure, GrammaticalStructureFactory, SemanticHeadFinder}
import edu.stanford.nlp.trees.{Tree => StanfordTree}
import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations
import CoreNLPUtils._
import edu.arizona.sista.discourse.rstparser.Utils._

/**
 * API for Stanford's CoreNLP tools
 * User: mihais
 * Date: 3/1/13
 */
class CoreNLPProcessor(internStrings:Boolean = true,
                       val withDiscourse:Boolean = false,
                       val maxSentenceLength:Int = 100) extends ShallowNLPProcessor(internStrings) {
  lazy val coref = mkCoref
  lazy val rstConstituentParser = CoreNLPProcessor.fetchParser(RSTParser.DEFAULT_CONSTITUENTSYNTAX_MODEL_PATH)

  //
  // we maintain our own copy of a LexicalizedParser to control which sentences are parsed
  // the CoreNLP option parser.maxlen does not work well
  //
  lazy val stanfordParser = mkLexicalizedParser
  lazy val gsf = mkGSF
  lazy val headFinder = new SemanticHeadFinder()

  def mkLexicalizedParser: LexicalizedParser = {
    val parser = LexicalizedParser.loadModel()
    parser
  }

  def mkGSF:GrammaticalStructureFactory = {
    val tlp = stanfordParser.getTLPParams.treebankLanguagePack
    val g = tlp.grammaticalStructureFactory(tlp.punctuationWordRejectFilter(), tlp.typedDependencyHeadFinder())
    g
  }

  def mkCoref: StanfordCoreNLP = {
    val props = new Properties()
    props.put("annotators", "dcoref")
    new StanfordCoreNLP(props, false)
  }

  override def parse(doc:Document) {
    val annotation = basicSanityCheck(doc)
    if (annotation.isEmpty) return

    val sas = annotation.get.get(classOf[SentencesAnnotation])
    var offset = 0
    for (sa <- sas) {
      // run the actual parser here
      val stanfordTree = stanfordParse(sa)

      // store Stanford annotations; Stanford dependencies are created here!
      ParserAnnotatorUtils.fillInParseAnnotations(false, true, gsf, sa, stanfordTree, GrammaticalStructure.Extras.NONE)

      // save our own structures
      if (stanfordTree != null) {
        // save the constituent tree, including head word information
        val position = new MutableNumber[Int](0)
        doc.sentences(offset).syntacticTree = Some(CoreNLPUtils.toTree(stanfordTree, headFinder, position))

        // save syntactic dependencies
        val basicDeps = sa.get(classOf[SemanticGraphCoreAnnotations.BasicDependenciesAnnotation])
        val collapsedDeps = sa.get(classOf[SemanticGraphCoreAnnotations.CollapsedCCProcessedDependenciesAnnotation])
        doc.sentences(offset).setDependencies(DependencyMap.STANFORD_BASIC, CoreNLPUtils.toDirectedGraph(basicDeps, in))
        doc.sentences(offset).setDependencies(DependencyMap.STANFORD_COLLAPSED, CoreNLPUtils.toDirectedGraph(collapsedDeps, in))
      } else {
        doc.sentences(offset).syntacticTree = None
      }
      offset += 1
    }
  }

  def stanfordParse(sentence:CoreMap):StanfordTree = {
    val constraints = sentence.get(classOf[ParserAnnotations.ConstraintAnnotation])
    val words = parensToSymbols(sentence.get(classOf[CoreAnnotations.TokensAnnotation]))

    var tree:StanfordTree = null

    //
    // Do not parse sentences longer than this
    // Those are likely coming from tables, so: (a) we don't know how to parse them anyway; (b) they would take forever
    //
    if(words.size < maxSentenceLength) {
      // the actual parsing
      val pq = stanfordParser.parserQuery()
      pq.setConstraints(constraints)

      //print("Parsing sentence:")
      //for(w <- words) print(s" ${w.toString()}")
      //println()

      pq.parse(words)

      // fetch the best tree
      try {
        tree = pq.getBestParse
        if (tree != null)
          tree.setScore(pq.getPCFGScore % -10000.0)
      } catch {
        case e: Exception =>
          System.err.println("WARNING: Parsing of sentence failed, possibly because of out of memory. " +
            "Will ignore and continue: " + edu.stanford.nlp.ling.Sentence.listToString(words))
      }

      //println("SYNTACTIC TREE: " + tree)
    } else {
      System.err.println("Skipping sentence of length " + words.size)
    }

    // create a fake tree if the actual parsing failed
    if(tree == null)
      tree = ParserUtils.xTree(words)

    //println("TREE: " + tree)
    tree
  }

  override def resolveCoreference(doc:Document) {
    val annotation = basicSanityCheck(doc)
    if (annotation.isEmpty) return

    if (doc.sentences.head.tags.isEmpty)
      throw new RuntimeException("ERROR: you have to run the POS tagger before coreference resolution!")
    if (doc.sentences.head.lemmas.isEmpty)
      throw new RuntimeException("ERROR: you have to run the lemmatizer before coreference resolution!")
    if (doc.sentences.head.entities.isEmpty)
      throw new RuntimeException("ERROR: you have to run the NER before coreference resolution!")
    if(doc.sentences.head.syntacticTree.isEmpty)
      throw new RuntimeException("ERROR: you have to run the parser before coreference resolution!")

    coref.annotate(annotation.get)

    val chains = annotation.get.get(classOf[CorefChainAnnotation])
    val mentions = new ListBuffer[CorefMention]

    for (cid <- chains.keySet()) {
      // println("cluster " + cid)
      val mentionMap = chains.get(cid).getMentionMap
      for (mid <- mentionMap.keySet()) {
        for (mention <- mentionMap.get(mid)) {
          // val isRep = mention == cluster.getRepresentativeMention
          // println("\tmention " + mid.getSource + " " + mid.getTarget + " " + mention.startIndex + " " + mention.endIndex + " " + isRep + " [" + mention.mentionSpan + "]")

          // Processor indexes things from 0 not 1!
          val m = new CorefMention(
            mid.getSource - 1,
            mid.getTarget - 1,
            mention.startIndex - 1,
            mention.endIndex - 1,
            cid)
          mentions += m
        }
      }
    }

    doc.coreferenceChains = Some(new CorefChains(mentions.toList))
  }

  override def discourse(doc:Document) {
    if(! withDiscourse) return
    basicSanityCheck(doc, checkAnnotation = false)

    if (doc.sentences.head.tags.isEmpty)
      throw new RuntimeException("ERROR: you have to run the POS tagger before discourse parsing!")
    if (doc.sentences.head.lemmas.isEmpty)
      throw new RuntimeException("ERROR: you have to run the lemmatizer before discourse parsing!")
    if(! hasDeps(doc.sentences.head))
      throw new RuntimeException("ERROR: you have to run the dependency parser before discourse parsing!")
    if(doc.sentences.head.syntacticTree.isEmpty)
      throw new RuntimeException("ERROR: you have to run the constituent parser before discourse parsing!")

    val out = rstConstituentParser.parse(doc)
    doc.discourseTree = Some(out._1)

    //println("FOUND DISCOURSE TREE:\n" + out._1)
  }
}

object CoreNLPProcessor {
  var rstParser:Option[RSTParser] = None

  def fetchParser(path:String):RSTParser = {
    this.synchronized {
      if(rstParser.isEmpty) rstParser = Some(RSTParser.loadFrom(path))
      rstParser.get
    }<commit_msg>Added CoreNLP tree-based sentiment analysis<commit_after>package edu.arizona.sista.processors.corenlp

import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations
import edu.stanford.nlp.trees.TreeCoreAnnotations.TreeAnnotation
import scala.collection.JavaConverters._
import edu.arizona.sista.discourse.rstparser.RSTParser
import edu.arizona.sista.processors._
import edu.arizona.sista.processors.shallownlp.ShallowNLPProcessor
import edu.arizona.sista.struct._
import edu.stanford.nlp.parser.lexparser.LexicalizedParser
import edu.stanford.nlp.parser.common.{ParserAnnotations, ParserUtils}
import edu.stanford.nlp.pipeline.{ParserAnnotatorUtils, StanfordCoreNLP}
import java.util.Properties
import collection.mutable.ListBuffer
import edu.stanford.nlp.ling.CoreAnnotations._
import scala.collection.JavaConversions._
import edu.stanford.nlp.util.CoreMap
import edu.stanford.nlp.ling.CoreAnnotations
import edu.stanford.nlp.dcoref.CorefCoreAnnotations.CorefChainAnnotation
import edu.stanford.nlp.trees.{GrammaticalStructure, GrammaticalStructureFactory, SemanticHeadFinder}
import edu.stanford.nlp.trees.{Tree => StanfordTree}
import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations
import edu.stanford.nlp.sentiment.SentimentCoreAnnotations
import edu.arizona.sista.discourse.rstparser.Utils._
import CoreNLPUtils._


/**
 * API for Stanford's CoreNLP tools
 * User: mihais
 * Date: 3/1/13
 */
class CoreNLPProcessor(
  internStrings:Boolean = true,
  val withDiscourse:Boolean = false,
  val maxSentenceLength:Int = 100
) extends ShallowNLPProcessor(internStrings) {

  lazy val coref = mkCoref
  lazy val rstConstituentParser = CoreNLPProcessor.fetchParser(RSTParser.DEFAULT_CONSTITUENTSYNTAX_MODEL_PATH)
  private lazy val sentimentAnalyzer = mkSentimentAnalyzer

  //
  // we maintain our own copy of a LexicalizedParser to control which sentences are parsed
  // the CoreNLP option parser.maxlen does not work well
  //
  lazy val stanfordParser = mkLexicalizedParser
  lazy val gsf = mkGSF
  lazy val headFinder = new SemanticHeadFinder()

  def mkLexicalizedParser: LexicalizedParser = {
    val parser = LexicalizedParser.loadModel()
    parser
  }

  def mkGSF:GrammaticalStructureFactory = {
    val tlp = stanfordParser.getTLPParams.treebankLanguagePack
    val g = tlp.grammaticalStructureFactory(tlp.punctuationWordRejectFilter(), tlp.typedDependencyHeadFinder())
    g
  }

  def mkCoref: StanfordCoreNLP = {
    val props = new Properties()
    props.put("annotators", "dcoref")
    new StanfordCoreNLP(props, false)
  }

  override def parse(doc:Document) {
    val annotation = basicSanityCheck(doc)
    if (annotation.isEmpty) return

    val sas = annotation.get.get(classOf[SentencesAnnotation])
    var offset = 0
    for (sa <- sas) {
      // run the actual parser here
      val stanfordTree = stanfordParse(sa)

      // store Stanford annotations; Stanford dependencies are created here!
      ParserAnnotatorUtils.fillInParseAnnotations(false, true, gsf, sa, stanfordTree, GrammaticalStructure.Extras.NONE)

      // save our own structures
      if (stanfordTree != null) {
        // save the constituent tree, including head word information
        val position = new MutableNumber[Int](0)
        doc.sentences(offset).syntacticTree = Some(CoreNLPUtils.toTree(stanfordTree, headFinder, position))

        // save syntactic dependencies
        val basicDeps = sa.get(classOf[SemanticGraphCoreAnnotations.BasicDependenciesAnnotation])
        val collapsedDeps = sa.get(classOf[SemanticGraphCoreAnnotations.CollapsedCCProcessedDependenciesAnnotation])
        doc.sentences(offset).setDependencies(DependencyMap.STANFORD_BASIC, CoreNLPUtils.toDirectedGraph(basicDeps, in))
        doc.sentences(offset).setDependencies(DependencyMap.STANFORD_COLLAPSED, CoreNLPUtils.toDirectedGraph(collapsedDeps, in))
      } else {
        doc.sentences(offset).syntacticTree = None
      }
      offset += 1
    }
  }

  def stanfordParse(sentence:CoreMap):StanfordTree = {
    val constraints = sentence.get(classOf[ParserAnnotations.ConstraintAnnotation])
    val words = parensToSymbols(sentence.get(classOf[CoreAnnotations.TokensAnnotation]))

    var tree:StanfordTree = null

    //
    // Do not parse sentences longer than this
    // Those are likely coming from tables, so: (a) we don't know how to parse them anyway; (b) they would take forever
    //
    if(words.size < maxSentenceLength) {
      // the actual parsing
      val pq = stanfordParser.parserQuery()
      pq.setConstraints(constraints)

      //print("Parsing sentence:")
      //for(w <- words) print(s" ${w.toString()}")
      //println()

      pq.parse(words)

      // fetch the best tree
      try {
        tree = pq.getBestParse
        if (tree != null)
          tree.setScore(pq.getPCFGScore % -10000.0)
      } catch {
        case e: Exception =>
          System.err.println("WARNING: Parsing of sentence failed, possibly because of out of memory. " +
            "Will ignore and continue: " + edu.stanford.nlp.ling.Sentence.listToString(words))
      }

      //println("SYNTACTIC TREE: " + tree)
    } else {
      System.err.println("Skipping sentence of length " + words.size)
    }

    // create a fake tree if the actual parsing failed
    if(tree == null)
      tree = ParserUtils.xTree(words)

    //println("TREE: " + tree)
    tree
  }

  override def resolveCoreference(doc:Document) {
    val annotation = basicSanityCheck(doc)
    if (annotation.isEmpty) return

    if (doc.sentences.head.tags.isEmpty)
      throw new RuntimeException("ERROR: you have to run the POS tagger before coreference resolution!")
    if (doc.sentences.head.lemmas.isEmpty)
      throw new RuntimeException("ERROR: you have to run the lemmatizer before coreference resolution!")
    if (doc.sentences.head.entities.isEmpty)
      throw new RuntimeException("ERROR: you have to run the NER before coreference resolution!")
    if(doc.sentences.head.syntacticTree.isEmpty)
      throw new RuntimeException("ERROR: you have to run the parser before coreference resolution!")

    coref.annotate(annotation.get)

    val chains = annotation.get.get(classOf[CorefChainAnnotation])
    val mentions = new ListBuffer[CorefMention]

    for (cid <- chains.keySet()) {
      // println("cluster " + cid)
      val mentionMap = chains.get(cid).getMentionMap
      for (mid <- mentionMap.keySet()) {
        for (mention <- mentionMap.get(mid)) {
          // val isRep = mention == cluster.getRepresentativeMention
          // println("\tmention " + mid.getSource + " " + mid.getTarget + " " + mention.startIndex + " " + mention.endIndex + " " + isRep + " [" + mention.mentionSpan + "]")

          // Processor indexes things from 0 not 1!
          val m = new CorefMention(
            mid.getSource - 1,
            mid.getTarget - 1,
            mention.startIndex - 1,
            mention.endIndex - 1,
            cid)
          mentions += m
        }
      }
    }

    doc.coreferenceChains = Some(new CorefChains(mentions.toList))
  }

  override def discourse(doc:Document) {
    if(! withDiscourse) return
    basicSanityCheck(doc, checkAnnotation = false)

    if (doc.sentences.head.tags.isEmpty)
      throw new RuntimeException("ERROR: you have to run the POS tagger before discourse parsing!")
    if (doc.sentences.head.lemmas.isEmpty)
      throw new RuntimeException("ERROR: you have to run the lemmatizer before discourse parsing!")
    if(! hasDeps(doc.sentences.head))
      throw new RuntimeException("ERROR: you have to run the dependency parser before discourse parsing!")
    if(doc.sentences.head.syntacticTree.isEmpty)
      throw new RuntimeException("ERROR: you have to run the constituent parser before discourse parsing!")

    val out = rstConstituentParser.parse(doc)
    doc.discourseTree = Some(out._1)

    //println("FOUND DISCOURSE TREE:\n" + out._1)
  }

  /** Create a pipeline for sentiment analysis */
  def mkSentimentAnalyzer: StanfordCoreNLP = {
    val props = new Properties()
    // NOTE: CoreNLP requires parse trees to be "binarized"
    // in order to be used for sentiment analysis
    //props.put("annotators", "sentiment")
    props.put("annotators", "binarizer, sentiment")
    props.put("customAnnotatorClass.binarizer", "edu.stanford.nlp.pipeline.BinarizerAnnotator")
    props.put("binarizer.tlppClass", "edu.stanford.nlp.parser.lexparser.EnglishTreebankParserParams")
    new StanfordCoreNLP(props, false)
  }

  /** Get a sentiment score from [[Document]] */
  def sentiment(doc: Document): Seq[Int] = for {
    s <- doc.sentences
  } yield sentiment(s)

  /** Get a sentiment score for a [[Sentence]] */
  def sentiment(s: Sentence): Int = {
    val a = sentenceToAnnotation(s)

    val sa = a.get(classOf[SentencesAnnotation]).asScala.toVector.head

    // TODO: convert existing parse back to Tree
    val tree = stanfordParse(sa)
    sa.set(classOf[TreeAnnotation], tree)
    // binarize tree
    //val binTree = ???
    //sa.set(classOf[BinarizedTreeAnnotation], binTree)

    sentimentAnalyzer.annotate(a)

    val sentimentTree = sa.get(classOf[SentimentCoreAnnotations.AnnotatedTree])
    val score = RNNCoreAnnotations.getPredictedClass(sentimentTree)
    score
  }

  /**
   * Get a Sentiment score for each [[Sentence]] in a span of text
   * @param text a String
   */
  def sentiment(text: String): Seq[Int] = {
    val doc = annotate(text)
    sentiment(doc)
  }
}

object CoreNLPProcessor {
  var rstParser:Option[RSTParser] = None

  def fetchParser(path:String):RSTParser = {
    this.synchronized {
      if(rstParser.isEmpty) rstParser = Some(RSTParser.loadFrom(path))
      rstParser.get
    }
{'fraction_non_alphanumeric': 0.09857387335995436, 'fraction_numerical': 0.002738163148887621, 'mean_word_length': 3.4744767738642164, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '681390', 'n_tokens_mistral': 5458, 'n_tokens_neox': 5311, 'n_words': 1229}