<commit_before>package edu.arizona.sista.odin

import scala.util.hashing.MurmurHash3._
import edu.arizona.sista.struct.Interval
import edu.arizona.sista.processors.Document
import edu.arizona.sista.odin.impl.StringMatcher

trait Mention extends Equals with Ordered[Mention]{
  /** A sequence of labels for this mention.
    * The first label in the sequence is considered the default.
    */
  def labels: Seq[String]

  /** The interval of token indicess that form this mention. */
  def tokenInterval: Interval

  /** The index of the sentence where this mention occurs. */
  def sentence: Int

  /** The document where this mention occurs. */
  def document: Document

  /** Should we report this mention at the end? */
  def keep: Boolean

  /** A map from argument name to a sequence of mentions.
    *
    * The value of the map is a sequence because there are events
    * that can have several arguments with the same name.
    * For example, in the biodomain, Binding may have several themes.
    */
  val arguments: Map[String, Seq[Mention]]

  /** default label */
  def label: String = labels.head

  /** name of matching rule */
  def foundBy: String

  /** index of the first token in the mention */
  def start: Int = tokenInterval.start

  /** one after the last token in the mention */
  def end: Int = tokenInterval.end

  def sentenceObj = document.sentences(sentence)

  /** character offset of the mention beginning */
  def startOffset: Int = sentenceObj.startOffsets(start)

  /** character offset of the mention end */
  def endOffset: Int = sentenceObj.endOffsets(end - 1)

  /** returns true if this is a valid mention */
  def isValid: Boolean = true

  /** returns true if the string matches any of the mention labels */
  def matches(label: String): Boolean = labels contains label

  /** returns true if the StringMatcher matches any of the mention labels */
  def matches(matcher: StringMatcher): Boolean = labels exists matcher.matches

  /** returns all tokens in mention */
  def words: Seq[String] = sentenceObj.words.slice(start, end)

  def tags: Option[Seq[String]] = sentenceObj.tags.map(_.slice(start, end))

  def lemmas: Option[Seq[String]] = sentenceObj.lemmas.map(_.slice(start, end))

  def entities: Option[Seq[String]] = sentenceObj.entities.map(_.slice(start, end))

  def chunks: Option[Seq[String]] = sentenceObj.chunks.map(_.slice(start, end))

  /** returns a string that contains the mention */
  def text: String = document.text match {
    case Some(txt) => txt.slice(startOffset, endOffset)
    case None =>
      // try to reconstruct the sentence using the character offsets
      val bits = words.head +: tokenInterval.toSeq.drop(1).map { i =>
        val spaces = " " * (sentenceObj.startOffsets(i) - sentenceObj.endOffsets(i - 1))
        val word = sentenceObj.words(i)
        spaces + word
      }
      bits.mkString
  }

  override def canEqual(a: Any) = a.isInstanceOf[Mention]

  override def equals(that: Any): Boolean = that match {
    case that: Mention => that.canEqual(this) && this.hashCode == that.hashCode
    case _ => false
  }

  def compare(that: Mention): Int = {
    require(this.document == that.document)

    (this.sentence - that.sentence,
      this.tokenInterval.start - that.tokenInterval.start,
      this.tokenInterval.end - that.tokenInterval.end) match {
      case (0,0,x) => x
      case (0,x,_) => x
      case (x,_,_) => x
    }
  }

  def precedes(that: Mention): Boolean = {
    this.compare(that) match {
      case c if c < 0 => true
      case _ => false
    }
  }

  override def hashCode: Int = {
    val h0 = stringHash("edu.arizona.sista.odin.Mention")
    val h1 = mix(h0, labels.hashCode)
    val h2 = mix(h1, tokenInterval.hashCode)
    val h3 = mix(h2, sentence.hashCode)
    val h4 = mix(h3, document.hashCode)
    val h5 = mixLast(h4, argumentsHashCode)
    finalizeHash(h5, 5)
  }

  private def argumentsHashCode: Int = {
    var h = stringHash("Mention.arguments")
    for ((name, args) <- arguments.toSeq.sortBy(_._1)) {
      h = mix(h, stringHash(name))
      h = mix(h, unorderedHash(args))
    }
    finalizeHash(h, arguments.size * 2)
  }
}

class TextBoundMention(
    val labels: Seq[String],
    val tokenInterval: Interval,
    val sentence: Int,
    val document: Document,
    val keep: Boolean,
    val foundBy: String
) extends Mention {

  def this(
    label: String,
    tokenInterval: Interval,
    sentence: Int,
    document: Document,
    keep: Boolean,
    foundBy: String
  ) = this(Seq(label), tokenInterval, sentence, document, keep, foundBy)

  // TextBoundMentions don't have arguments
  val arguments: Map[String, Seq[Mention]] = Map.empty
}

class EventMention(
    val labels: Seq[String],
    val trigger: TextBoundMention,
    val arguments: Map[String, Seq[Mention]],
    val sentence: Int,
    val document: Document,
    val keep: Boolean,
    val foundBy: String
) extends Mention {

  //require(arguments.values.flatten.nonEmpty, "EventMentions need arguments")

  def this(
    label: String,<commit_msg>More nuanced approach to the prior commit<commit_after>package edu.arizona.sista.odin

import scala.util.hashing.MurmurHash3._
import edu.arizona.sista.struct.Interval
import edu.arizona.sista.processors.Document
import edu.arizona.sista.odin.impl.StringMatcher

trait Mention extends Equals with Ordered[Mention]{
  /** A sequence of labels for this mention.
    * The first label in the sequence is considered the default.
    */
  def labels: Seq[String]

  /** The interval of token indicess that form this mention. */
  def tokenInterval: Interval

  /** The index of the sentence where this mention occurs. */
  def sentence: Int

  /** The document where this mention occurs. */
  def document: Document

  /** Should we report this mention at the end? */
  def keep: Boolean

  /** A map from argument name to a sequence of mentions.
    *
    * The value of the map is a sequence because there are events
    * that can have several arguments with the same name.
    * For example, in the biodomain, Binding may have several themes.
    */
  val arguments: Map[String, Seq[Mention]]

  /** default label */
  def label: String = labels.head

  /** name of matching rule */
  def foundBy: String

  /** index of the first token in the mention */
  def start: Int = tokenInterval.start

  /** one after the last token in the mention */
  def end: Int = tokenInterval.end

  def sentenceObj = document.sentences(sentence)

  /** character offset of the mention beginning */
  def startOffset: Int = sentenceObj.startOffsets(start)

  /** character offset of the mention end */
  def endOffset: Int = sentenceObj.endOffsets(end - 1)

  /** returns true if this is a valid mention */
  def isValid: Boolean = true

  /** returns true if the string matches any of the mention labels */
  def matches(label: String): Boolean = labels contains label

  /** returns true if the StringMatcher matches any of the mention labels */
  def matches(matcher: StringMatcher): Boolean = labels exists matcher.matches

  /** returns all tokens in mention */
  def words: Seq[String] = sentenceObj.words.slice(start, end)

  def tags: Option[Seq[String]] = sentenceObj.tags.map(_.slice(start, end))

  def lemmas: Option[Seq[String]] = sentenceObj.lemmas.map(_.slice(start, end))

  def entities: Option[Seq[String]] = sentenceObj.entities.map(_.slice(start, end))

  def chunks: Option[Seq[String]] = sentenceObj.chunks.map(_.slice(start, end))

  /** returns a string that contains the mention */
  def text: String = document.text match {
    case Some(txt) => txt.slice(startOffset, endOffset)
    case None =>
      // try to reconstruct the sentence using the character offsets
      val bits = words.head +: tokenInterval.toSeq.drop(1).map { i =>
        val spaces = " " * (sentenceObj.startOffsets(i) - sentenceObj.endOffsets(i - 1))
        val word = sentenceObj.words(i)
        spaces + word
      }
      bits.mkString
  }

  override def canEqual(a: Any) = a.isInstanceOf[Mention]

  override def equals(that: Any): Boolean = that match {
    case that: Mention => that.canEqual(this) && this.hashCode == that.hashCode
    case _ => false
  }

  def compare(that: Mention): Int = {
    require(this.document == that.document)

    (this.sentence - that.sentence,
      this.tokenInterval.start - that.tokenInterval.start,
      this.tokenInterval.end - that.tokenInterval.end) match {
      case (0,0,x) => x
      case (0,x,_) => x
      case (x,_,_) => x
    }
  }

  def precedes(that: Mention): Boolean = {
    this.compare(that) match {
      case c if c < 0 => true
      case _ => false
    }
  }

  override def hashCode: Int = {
    val h0 = stringHash("edu.arizona.sista.odin.Mention")
    val h1 = mix(h0, labels.hashCode)
    val h2 = mix(h1, tokenInterval.hashCode)
    val h3 = mix(h2, sentence.hashCode)
    val h4 = mix(h3, document.hashCode)
    val h5 = mixLast(h4, argumentsHashCode)
    finalizeHash(h5, 5)
  }

  private def argumentsHashCode: Int = {
    var h = stringHash("Mention.arguments")
    for ((name, args) <- arguments.toSeq.sortBy(_._1)) {
      h = mix(h, stringHash(name))
      h = mix(h, unorderedHash(args))
    }
    finalizeHash(h, arguments.size * 2)
  }
}

class TextBoundMention(
    val labels: Seq[String],
    val tokenInterval: Interval,
    val sentence: Int,
    val document: Document,
    val keep: Boolean,
    val foundBy: String
) extends Mention {

  def this(
    label: String,
    tokenInterval: Interval,
    sentence: Int,
    document: Document,
    keep: Boolean,
    foundBy: String
  ) = this(Seq(label), tokenInterval, sentence, document, keep, foundBy)

  // TextBoundMentions don't have arguments
  val arguments: Map[String, Seq[Mention]] = Map.empty
}

class EventMention(
    val labels: Seq[String],
    val trigger: TextBoundMention,
    val arguments: Map[String, Seq[Mention]],
    val sentence: Int,
    val document: Document,
    val keep: Boolean,
    val foundBy: String
) extends Mention {

  require(arguments.values.flatten.nonEmpty || labels.contains("Unresolved"), "EventMentions need arguments")

  def this(
    label: String,
{'fraction_non_alphanumeric': 0.10997542997542997, 'fraction_numerical': 0.004520884520884521, 'mean_word_length': 3.198019801980198, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3013274', 'n_tokens_mistral': 3168, 'n_tokens_neox': 3071, 'n_words': 1000}