<commit_before>package edu.arizona.sista.processors

import edu.arizona.sista.discourse.rstparser.DiscourseTree
import edu.arizona.sista.struct.{Tree, DirectedGraph}
import DependencyMap._

import collection.mutable
import collection.mutable.ListBuffer
import java.lang.StringBuilder

/**
 * Stores all annotations for one document
 * User: mihais
 * Date: 3/1/13
 */
class Document( var id:Option[String],
                val sentences:Array[Sentence],
                var coreferenceChains:Option[CorefChains],
                var discourseTree:Option[DiscourseTree],
                var text:Option[String]) extends Serializable {

  def this(sa:Array[Sentence], cc:Option[CorefChains], dt:Option[DiscourseTree]) {
    this(None, sa, cc, dt, None)
  }

  def this(sa: Array[Sentence]) {
    this(None, sa, None, None, None)
  }

  /** Clears any internal state potentially constructed by the annotators */
  def clear() { }
}

/** Stores the annotations for a single sentence */
class Sentence(
                /** Actual tokens in this sentence */
                val words:Array[String],
                /** Start character offsets for the words; start at 0 */
                val startOffsets:Array[Int],
                /** End character offsets for the words; start at 0 */
                val endOffsets:Array[Int],
                /** POS tags for words */
                var tags:Option[Array[String]],
                /** Lemmas */
                var lemmas:Option[Array[String]],
                /** NE labels */
                var entities:Option[Array[String]],
                /** Normalized values of named/numeric entities, such as dates */
                var norms:Option[Array[String]],
                /** Shallow parsing labels */
                var chunks:Option[Array[String]],
                /** Constituent tree of this sentence; includes head words */
                var syntacticTree:Option[Tree],
                /** DAG of syntactic dependencies; word offsets start at 0 */
                var dependenciesByType:DependencyMap) extends Serializable {

  def this(
            words:Array[String],
            startOffsets:Array[Int],
            endOffsets:Array[Int]) =
    this(words, startOffsets, endOffsets,
      None, None, None, None, None, None, new DependencyMap)

  def size:Int = words.length

  /**
   * Default dependencies: first Stanford collapsed, then Stanford basic, then None
   * @return A directed graph of dependencies if any exist, otherwise None
   */
  def dependencies:Option[DirectedGraph[String]] = {
    if(dependenciesByType == null) return None

    if(dependenciesByType.contains(STANFORD_COLLAPSED))
      dependenciesByType.get(STANFORD_COLLAPSED)
    else if(dependenciesByType.contains(STANFORD_BASIC))
      dependenciesByType.get(STANFORD_BASIC)
    else
      None
  }

  /** Fetches the Stanford basic dependencies */
  def stanfordBasicDependencies:Option[DirectedGraph[String]] = {
    if(dependenciesByType == null) return None
    dependenciesByType.get(STANFORD_BASIC)
  }

  /** Fetches the Stanford collapsed dependencies */
  def stanfordCollapsedDependencies:Option[DirectedGraph[String]] = {
    if(dependenciesByType == null) return None
    dependenciesByType.get(STANFORD_COLLAPSED)
  }

  def setDependencies(depType:Int, deps:DirectedGraph[String]): Unit = {
    if(dependenciesByType == null)
      dependenciesByType = new DependencyMap
    dependenciesByType += (depType -> deps)
  }

  /**
   * Recreates the text of the sentence, preserving the original number of white spaces between tokens
   * @return the text of the sentence
   */
  def getSentenceText:String =  getSentenceFragmentText(0, words.length)

  def getSentenceFragmentText(start:Int, end:Int):String = {
    // optimize the single token case
    if(end - start == 1) words(start)

    val text = new mutable.StringBuilder()
    for(i <- start until end) {
      if(i > start) {
        // add as many white spaces as recorded between tokens
        // sometimes this space is negative: in BioNLPProcessor we replace "/" with "and"
        //   in these cases, let's make sure we print 1 space, otherwise the text is hard to read
        val numberOfSpaces = math.max(1, startOffsets(i) - endOffsets(i - 1))
        for (j <- 0 until numberOfSpaces) {
          text.append(" ")
        }
      }
      text.append(words(i))
    }
    text.toString()<commit_msg>Allow getSentenceText() to be parameterless again<commit_after>package edu.arizona.sista.processors

import edu.arizona.sista.discourse.rstparser.DiscourseTree
import edu.arizona.sista.struct.{Tree, DirectedGraph}
import DependencyMap._

import collection.mutable
import collection.mutable.ListBuffer
import java.lang.StringBuilder

/**
 * Stores all annotations for one document
 * User: mihais
 * Date: 3/1/13
 */
class Document( var id:Option[String],
                val sentences:Array[Sentence],
                var coreferenceChains:Option[CorefChains],
                var discourseTree:Option[DiscourseTree],
                var text:Option[String]) extends Serializable {

  def this(sa:Array[Sentence], cc:Option[CorefChains], dt:Option[DiscourseTree]) {
    this(None, sa, cc, dt, None)
  }

  def this(sa: Array[Sentence]) {
    this(None, sa, None, None, None)
  }

  /** Clears any internal state potentially constructed by the annotators */
  def clear() { }
}

/** Stores the annotations for a single sentence */
class Sentence(
                /** Actual tokens in this sentence */
                val words:Array[String],
                /** Start character offsets for the words; start at 0 */
                val startOffsets:Array[Int],
                /** End character offsets for the words; start at 0 */
                val endOffsets:Array[Int],
                /** POS tags for words */
                var tags:Option[Array[String]],
                /** Lemmas */
                var lemmas:Option[Array[String]],
                /** NE labels */
                var entities:Option[Array[String]],
                /** Normalized values of named/numeric entities, such as dates */
                var norms:Option[Array[String]],
                /** Shallow parsing labels */
                var chunks:Option[Array[String]],
                /** Constituent tree of this sentence; includes head words */
                var syntacticTree:Option[Tree],
                /** DAG of syntactic dependencies; word offsets start at 0 */
                var dependenciesByType:DependencyMap) extends Serializable {

  def this(
            words:Array[String],
            startOffsets:Array[Int],
            endOffsets:Array[Int]) =
    this(words, startOffsets, endOffsets,
      None, None, None, None, None, None, new DependencyMap)

  def size:Int = words.length

  /**
   * Default dependencies: first Stanford collapsed, then Stanford basic, then None
   * @return A directed graph of dependencies if any exist, otherwise None
   */
  def dependencies:Option[DirectedGraph[String]] = {
    if(dependenciesByType == null) return None

    if(dependenciesByType.contains(STANFORD_COLLAPSED))
      dependenciesByType.get(STANFORD_COLLAPSED)
    else if(dependenciesByType.contains(STANFORD_BASIC))
      dependenciesByType.get(STANFORD_BASIC)
    else
      None
  }

  /** Fetches the Stanford basic dependencies */
  def stanfordBasicDependencies:Option[DirectedGraph[String]] = {
    if(dependenciesByType == null) return None
    dependenciesByType.get(STANFORD_BASIC)
  }

  /** Fetches the Stanford collapsed dependencies */
  def stanfordCollapsedDependencies:Option[DirectedGraph[String]] = {
    if(dependenciesByType == null) return None
    dependenciesByType.get(STANFORD_COLLAPSED)
  }

  def setDependencies(depType:Int, deps:DirectedGraph[String]): Unit = {
    if(dependenciesByType == null)
      dependenciesByType = new DependencyMap
    dependenciesByType += (depType -> deps)
  }

  /**
   * Recreates the text of the sentence, preserving the original number of white spaces between tokens
   * @return the text of the sentence
   */
  def getSentenceText():String =  getSentenceFragmentText(0, words.length)

  def getSentenceFragmentText(start:Int, end:Int):String = {
    // optimize the single token case
    if(end - start == 1) words(start)

    val text = new mutable.StringBuilder()
    for(i <- start until end) {
      if(i > start) {
        // add as many white spaces as recorded between tokens
        // sometimes this space is negative: in BioNLPProcessor we replace "/" with "and"
        //   in these cases, let's make sure we print 1 space, otherwise the text is hard to read
        val numberOfSpaces = math.max(1, startOffsets(i) - endOffsets(i - 1))
        for (j <- 0 until numberOfSpaces) {
          text.append(" ")
        }
      }
      text.append(words(i))
    }
    text.toString()
{'fraction_non_alphanumeric': 0.09831271091113611, 'fraction_numerical': 0.002924634420697413, 'mean_word_length': 2.670933113129645, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '825639', 'n_tokens_mistral': 2567, 'n_tokens_neox': 2410, 'n_words': 690}