<commit_before>package edu.arizona.sista.odin

import org.scalatest._
import edu.arizona.sista.processors.bionlp.BioNLPProcessor
import edu.arizona.sista.struct.Interval
import edu.arizona.sista.odin.impl.TokenPattern
import edu.arizona.sista.odin._

class TestTokenPattern extends FlatSpec with Matchers {
  val proc = new BioNLPProcessor
  val text1 = "TGFBR2 phosphorylates peri-kappa B and inhibits the ubiquitination of SMAD3."
  val text2 = "TGFBR2 phosphorylates peri-kappa B and peri-kappa C and inhibits the ubiquitination of SMAD3."
  val text3 = "TGFBR2 phosphorylates peri-kappa B and peri-kappa C by TGFBR3 and inhibits the ubiquitination of SMAD3."

  text1 should "contain one match" in {
    val doc = proc annotate text1
    val p = TokenPattern.compile("@Phosphorylation and inhibits")
    val mentions = Seq(new TextBoundMention("Phosphorylation", Interval(0, 4), 0, doc, true, "<MANUAL>"))
    val state = State(mentions)
    val results = p.findAllIn(0, doc, state)
    results should have size (1)
  }

  text2 should "contain one match" in {
    val doc = proc annotate text2
    val p = TokenPattern.compile("@Phosphorylation and inhibits")
    val mentions = Seq(new TextBoundMention("Phosphorylation", Interval(0, 4), 0, doc, true, "<MANUAL>"),
                       new TextBoundMention("Phosphorylation", Interval(0, 7), 0, doc, true, "<MANUAL>"))
    val state = State(mentions)
    val results = p.findAllIn(0, doc, state)
    results should have size (1)
  }

  text3 should "contain two matches" in {
    val doc = proc annotate text3
    val p = TokenPattern.compile("@Phosphorylation and inhibits")
    val mentions = Seq(new TextBoundMention("Phosphorylation", Interval(0, 9), 0, doc, true, "<MANUAL>"),
                       new TextBoundMention("Phosphorylation", Interval(0, 9), 0, doc, true, "<MANUAL>"))
    val state = State(mentions)
    val results = p.findAllIn(0, doc, state)
    results should have size (2)
  }

  val text4 = "a b c d e f g h i c"
  val doc = proc annotate text4

  text4 should "match with a lazy plus" in {
    val p = TokenPattern.compile("a []+? c")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (0),
      'end (3)
    )
  }

  it should "match with a greedy plus" in {
    val p = TokenPattern.compile("a []+ c")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (0),
      'end (10)
    )
  }

  it should "match with a lazy star" in {
    val p = TokenPattern.compile("a []*? c")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (0),
      'end (3)
    )
  }

  it should "match with a greedy star" in {
    val p = TokenPattern.compile("a []* c")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (0),
      'end (10)
    )
  }

  val text5 = "JAK3 phosphorylates three HuR residues (Y63, Y68, Y200)"
  val doc5 = proc annotate text5

  text5 should "match Y200 using greedy plus" in {
    val p = TokenPattern.compile("[]+ @site:Site")
    val mentions = Seq(
      new TextBoundMention("Site", Interval(6), 0, doc5, true, "<MANUAL>"),
      new TextBoundMention("Site", Interval(8), 0, doc5, true, "<MANUAL>"),<commit_msg>added tests<commit_after>package edu.arizona.sista.odin

import org.scalatest._
import edu.arizona.sista.processors.bionlp.BioNLPProcessor
import edu.arizona.sista.struct.Interval
import edu.arizona.sista.odin.impl.TokenPattern
import edu.arizona.sista.odin._

class TestTokenPattern extends FlatSpec with Matchers {
  val proc = new BioNLPProcessor
  val text1 = "TGFBR2 phosphorylates peri-kappa B and inhibits the ubiquitination of SMAD3."
  val text2 = "TGFBR2 phosphorylates peri-kappa B and peri-kappa C and inhibits the ubiquitination of SMAD3."
  val text3 = "TGFBR2 phosphorylates peri-kappa B and peri-kappa C by TGFBR3 and inhibits the ubiquitination of SMAD3."

  text1 should "contain one match" in {
    val doc = proc annotate text1
    val p = TokenPattern.compile("@Phosphorylation and inhibits")
    val mentions = Seq(new TextBoundMention("Phosphorylation", Interval(0, 4), 0, doc, true, "<MANUAL>"))
    val state = State(mentions)
    val results = p.findAllIn(0, doc, state)
    results should have size (1)
  }

  text2 should "contain one match" in {
    val doc = proc annotate text2
    val p = TokenPattern.compile("@Phosphorylation and inhibits")
    val mentions = Seq(new TextBoundMention("Phosphorylation", Interval(0, 4), 0, doc, true, "<MANUAL>"),
                       new TextBoundMention("Phosphorylation", Interval(0, 7), 0, doc, true, "<MANUAL>"))
    val state = State(mentions)
    val results = p.findAllIn(0, doc, state)
    results should have size (1)
  }

  text3 should "contain two matches" in {
    val doc = proc annotate text3
    val p = TokenPattern.compile("@Phosphorylation and inhibits")
    val mentions = Seq(new TextBoundMention("Phosphorylation", Interval(0, 9), 0, doc, true, "<MANUAL>"),
                       new TextBoundMention("Phosphorylation", Interval(0, 9), 0, doc, true, "<MANUAL>"))
    val state = State(mentions)
    val results = p.findAllIn(0, doc, state)
    results should have size (2)
  }

  val text4 = "a b c d e f g h i c"
  val doc = proc annotate text4

  text4 should "match with a lazy plus" in {
    val p = TokenPattern.compile("a []+? c")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (0),
      'end (3)
    )
  }

  it should "match with a greedy plus" in {
    val p = TokenPattern.compile("a []+ c")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (0),
      'end (10)
    )
  }

  it should "match with a lazy star" in {
    val p = TokenPattern.compile("a []*? c")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (0),
      'end (3)
    )
  }

  it should "match with a greedy star" in {
    val p = TokenPattern.compile("a []* c")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (0),
      'end (10)
    )
  }

  it should "match with postitive lookbehind" in {
    val p = TokenPattern.compile("(?<=a) b")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (1),
      'end (2)
    )
  }

  it should "not match with negative lookbehind" in {
    val p = TokenPattern.compile("(?<!a) b")
    val results = p.findAllIn(0, doc, None)
    results should be ('empty)
  }

  it should "match with postitive lookahead" in {
    val p = TokenPattern.compile("b (?=c)")
    val results = p.findAllIn(0, doc, None)
    results should have size (1)
    results.head.interval should have (
      'start (1),
      'end (2)
    )
  }

  it should "not match with negative lookahead" in {
    val p = TokenPattern.compile("b (?!c)")
    val results = p.findAllIn(0, doc, None)
    results should be ('empty)
  }


  val text5 = "JAK3 phosphorylates three HuR residues (Y63, Y68, Y200)"
  val doc5 = proc annotate text5

  text5 should "match Y200 using greedy plus" in {
    val p = TokenPattern.compile("[]+ @site:Site")
    val mentions = Seq(
      new TextBoundMention("Site", Interval(6), 0, doc5, true, "<MANUAL>"),
      new TextBoundMention("Site", Interval(8), 0, doc5, true, "<MANUAL>"),
{'fraction_non_alphanumeric': 0.11717433570965652, 'fraction_numerical': 0.021775761503564486, 'mean_word_length': 3.128410914927769, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3313764', 'n_tokens_mistral': 2719, 'n_tokens_neox': 2570, 'n_words': 850}