<commit_before>package edu.arizona.sista.matcher

import edu.arizona.sista.struct.Interval
import edu.arizona.sista.processors.Document

trait DependencyPatternParsers extends TokenPatternParsers {
  // comments are considered whitespace
  override val whiteSpace = """([ \t\x0B\f\r]|#.*)+""".r
  val eol = "\n"

  def dependencyPattern: Parser[DependencyPattern] =
    triggerFinder ~ rep1(eol) ~ repsep(argPattern, rep1(eol)) ^^ {
      case trigger ~ _ ~ arguments => new DependencyPattern(trigger, arguments)
    }

  def triggerFinder: Parser[TokenPattern] = "trigger" ~> ":" ~> tokenPattern

  def argPattern: Parser[ArgumentPattern] =
    ident ~ opt("?") ~ ":" ~ disjunctiveDepPattern ^^ {
      case "trigger" ~ _ ~ _ ~ _ => sys.error("`trigger` is not a valid argument name")
      case name ~ None ~ _ ~ pat => new ArgumentPattern(name, pat, true)
      case name ~ Some("?") ~ _ ~ pat => new ArgumentPattern(name, pat, false)
    }

  def disjunctiveDepPattern: Parser[DependencyPatternNode] =
    concatDepPattern ~ rep("|" ~> concatDepPattern) ^^ {
      case first ~ rest => (first /: rest) {
        case (lhs, rhs) => new DisjunctiveDependencyPattern(lhs, rhs)
      }
    }

  def concatDepPattern: Parser[DependencyPatternNode] =
    filteredDepPattern ~ rep(filteredDepPattern) ^^ {
      case first ~ rest => (first /: rest) {
        case (lhs, rhs) => new ConcatDependencyPattern(lhs, rhs)
      }
    }

  def filteredDepPattern: Parser[DependencyPatternNode] =
    quantifiedDepPattern ~ opt(tokenConstraint) ^^ {
      case pat ~ None => pat
      case pat ~ Some(constraint) => new FilteredDependencyPattern(pat, constraint)
    }

  def quantifiedDepPattern: Parser[DependencyPatternNode] =
    atomicDepPattern ~ opt("?"|"*"|"+") ^^ {
      case pat ~ None => pat
      case pat ~ Some("?") => new OptionalDependencyPattern(pat)
      case pat ~ Some("*") => new KleeneDependencyPattern(pat)
      case pat ~ Some("+") => new ConcatDependencyPattern(pat, new KleeneDependencyPattern(pat))
    }

  def atomicDepPattern: Parser[DependencyPatternNode] =
    outgoingDepPattern | incomingDepPattern | "(" ~> disjunctiveDepPattern <~ ")"

  def outgoingDepPattern: Parser[DependencyPatternNode] =
    opt(">") ~> stringMatcher ^^ { new OutgoingDependencyPattern(_) }

  def incomingDepPattern: Parser[DependencyPatternNode] =
    "<" ~> stringMatcher ^^ { new IncomingDependencyPattern(_) }
}<commit_msg>added range quantifiers to dependency patterns<commit_after>package edu.arizona.sista.matcher

import edu.arizona.sista.struct.Interval
import edu.arizona.sista.processors.Document

trait DependencyPatternParsers extends TokenPatternParsers {
  // comments are considered whitespace
  override val whiteSpace = """([ \t\x0B\f\r]|#.*)+""".r
  val eol = "\n"

  def dependencyPattern: Parser[DependencyPattern] =
    triggerFinder ~ rep1(eol) ~ repsep(argPattern, rep1(eol)) ^^ {
      case trigger ~ _ ~ arguments => new DependencyPattern(trigger, arguments)
    }

  def triggerFinder: Parser[TokenPattern] = "trigger" ~> ":" ~> tokenPattern

  def argPattern: Parser[ArgumentPattern] =
    ident ~ opt("?") ~ ":" ~ disjunctiveDepPattern ^^ {
      case "trigger" ~ _ ~ _ ~ _ => sys.error("`trigger` is not a valid argument name")
      case name ~ None ~ _ ~ pat => new ArgumentPattern(name, pat, true)
      case name ~ Some("?") ~ _ ~ pat => new ArgumentPattern(name, pat, false)
    }

  def disjunctiveDepPattern: Parser[DependencyPatternNode] =
    concatDepPattern ~ rep("|" ~> concatDepPattern) ^^ {
      case first ~ rest => (first /: rest) {
        case (lhs, rhs) => new DisjunctiveDependencyPattern(lhs, rhs)
      }
    }

  def concatDepPattern: Parser[DependencyPatternNode] =
    filteredDepPattern ~ rep(filteredDepPattern) ^^ {
      case first ~ rest => (first /: rest) {
        case (lhs, rhs) => new ConcatDependencyPattern(lhs, rhs)
      }
    }

  def filteredDepPattern: Parser[DependencyPatternNode] =
    (repeatDepPattern | rangeDepPattern | quantifiedDepPattern | atomicDepPattern) ~ opt(tokenConstraint) ^^ {
      case pat ~ None => pat
      case pat ~ Some(constraint) => new FilteredDependencyPattern(pat, constraint)
    }

  def quantifiedDepPattern: Parser[DependencyPatternNode] =
    atomicDepPattern ~ ("?"|"*"|"+") ^^ {
      case pat ~ "?" => new OptionalDependencyPattern(pat)
      case pat ~ "*" => new KleeneDependencyPattern(pat)
      case pat ~ "+" => new ConcatDependencyPattern(pat, new KleeneDependencyPattern(pat))
    }

  // helper function that repeats a pattern N times
  private def repeatPattern(pattern: DependencyPatternNode, n: Int): DependencyPatternNode = {
    require(n > 0, "'n' must be greater than zero")
    (pattern /: Seq.fill(n - 1)(pattern)) {
      case (lhs, rhs) => new ConcatDependencyPattern(lhs, rhs)
    }
  }

  def repeatDepPattern: Parser[DependencyPatternNode] = atomicDepPattern ~ "{" ~ int ~ "}" ^^ {
    case pat ~ _ ~ n ~ _ => repeatPattern(pat, n)
  }

  def rangeDepPattern: Parser[DependencyPatternNode] = atomicDepPattern ~ "{" ~ opt(int) ~ "," ~ opt(int) ~ "}" ^^ {
    case pat ~ _ ~ from ~ _ ~ to ~ _ => (from, to) match {
      case (None, None) =>
        sys.error("invalid range")
      case (None, Some(n)) =>
        repeatPattern(new OptionalDependencyPattern(pat), n)
      case (Some(m), None) =>
        val req = repeatPattern(pat, m)
        val kleene = new KleeneDependencyPattern(pat)
        new ConcatDependencyPattern(req, kleene)
      case (Some(m), Some(n)) =>
        require(n > m, "'to' must be greater than 'from'")
        val req = repeatPattern(pat, m)
        val opt = repeatPattern(new OptionalDependencyPattern(pat), n - m)
        new ConcatDependencyPattern(req, opt)
    }
  }

  def atomicDepPattern: Parser[DependencyPatternNode] =
    outgoingDepPattern | incomingDepPattern | "(" ~> disjunctiveDepPattern <~ ")"

  def outgoingDepPattern: Parser[DependencyPatternNode] =
    opt(">") ~> stringMatcher ^^ { new OutgoingDependencyPattern(_) }

  def incomingDepPattern: Parser[DependencyPatternNode] =
    "<" ~> stringMatcher ^^ { new IncomingDependencyPattern(_) }
}
{'fraction_non_alphanumeric': 0.13370291863688244, 'fraction_numerical': 0.0013044187184086092, 'mean_word_length': 3.7403400309119013, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3112964', 'n_tokens_mistral': 1900, 'n_tokens_neox': 1859, 'n_words': 430}