<commit_before>package edu.arizona.sista.matcher

import scala.annotation.tailrec
import scala.util.matching.Regex
import scala.util.parsing.combinator._
import edu.arizona.sista.struct.Interval
import edu.arizona.sista.processors.{Sentence, Document}



object StringMatcherParser extends RegexParsers {
  def parse(input: String): Prog = parseAll(pattern, input) match {
    case Success(result, _) => result
    case failure: NoSuccess => sys.error(failure.msg)
  }

  def int: Parser[Int] = """\d+""".r ^^ { _.toInt }

  def ident: Parser[String] =
    """\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*""".r

  // single- or double-quote delimited string literal
  def quotedLiteral: Parser[String] =
    """'[^\\']*(?:\\.[^\\']*)*'|"[^\\"]*(?:\\.[^\\"]*)*"""".r ^^ {
      case s => """\\(.)""".r.replaceAllIn(s.drop(1).dropRight(1), m => m.group(1))
    }

  def stringLiteral: Parser[String] = ident | quotedLiteral

  // match a perl style "/" delimited regular expression
  // "\" is the escape character, so "\/" becomes "/"
  def regexLiteral: Parser[Regex] = """/[^\\/]*(?:\\.[^\\/]*)*/""".r ^^ {
    case s => s.drop(1).dropRight(1).replaceAll("""\\/""", "/").r
  }

  def exactStringMatcher: Parser[StringMatcher] = stringLiteral ^^ {
    case string => new ExactStringMatcher(string)
  }

  def regexStringMatcher: Parser[StringMatcher] = regexLiteral ^^ {
    case regex => new RegexStringMatcher(regex)
  }

  def stringMatcher: Parser[StringMatcher] = exactStringMatcher | regexStringMatcher

  def fieldConstraint: Parser[TokenConstraint] = ident ~ "=" ~ stringMatcher ^^ {
    case "word" ~ _ ~ matcher => new WordConstraint(matcher)
    case "lemma" ~ _ ~ matcher => new LemmaConstraint(matcher)
    case "tag" ~ _ ~ matcher => new TagConstraint(matcher)
    case "entity" ~ _ ~ matcher => new EntityConstraint(matcher)
    case "chunk" ~ _ ~ matcher => new ChunkConstraint(matcher)
    case _ => sys.error("unrecognized token field")
  }

  def atomicConstraint: Parser[TokenConstraint] = fieldConstraint | "(" ~> disjunctiveConstraint <~ ")"

  def negatedConstraint: Parser[TokenConstraint] = opt("!") ~ atomicConstraint ^^ {
    case None ~ constraint => constraint
    case Some("!") ~ constraint => new NegatedConstraint(constraint)
  }

  def conjunctiveConstraint: Parser[TokenConstraint] = negatedConstraint ~ rep("&" ~> negatedConstraint) ^^ {
    case first ~ rest => (first /: rest) {
      case (lhs, rhs) => new ConjunctiveConstraint(lhs, rhs)
    }
  }

  def disjunctiveConstraint: Parser[TokenConstraint] = conjunctiveConstraint ~ rep("|" ~> conjunctiveConstraint) ^^ {
    case first ~ rest => (first /: rest) {
      case (lhs, rhs) => new DisjunctiveConstraint(lhs, rhs)
    }
  }

  def wordConstraint: Parser[TokenConstraint] = stringMatcher ^^ {
    case matcher => new WordConstraint(matcher)
  }

  def tokenConstraint: Parser[TokenConstraint] = "[" ~> disjunctiveConstraint <~ "]"

  def tokenPattern: Parser[Frag] = (wordConstraint | tokenConstraint) ^^ {
    case constraint => Frag(Match(constraint))
  }

  def capturePattern: Parser[Frag] = "(?<" ~ ident ~ ">" ~ splitPattern ~ ")" ^^ {
    case _ ~ name ~ _ ~ frag ~ _ => capture(name, frag)
  }

  def atomicPattern: Parser[Frag] = tokenPattern | capturePattern

  def repeatedPattern: Parser[Frag] = atomicPattern ~ ("??"|"*?"|"+?"|"?"|"*"|"+") ^^ {
    case pattern ~ "?" => greedyOptional(pattern)
    case pattern ~ "??" => lazyOptional(pattern)
    case pattern ~ "*" => greedyKleene(pattern)
    case pattern ~ "*?" => lazyKleene(pattern)
    case pattern ~ "+" => Frag(pattern, greedyKleene(pattern))
    case pattern ~ "+?" => Frag(pattern, lazyKleene(pattern))
  }

  def rangePattern: Parser[Frag] = atomicPattern ~ "{" ~ int ~ "," ~ int ~ "}" ^^ {
    case frag ~ _ ~ from ~ _ ~ to ~ _ => greedyRange(frag, Some(from), Some(to))
  }

  def fromPattern: Parser[Frag] = atomicPattern ~ "{" ~ int ~ "," ~ "}" ^^ {
    case frag ~ _ ~ from ~ _ ~ _ => greedyRange(frag, Some(from), None)
  }

  def toPattern: Parser[Frag] = atomicPattern ~ "{" ~ "," ~ int ~ "}" ^^ {
    case frag ~ _ ~ _ ~ to ~ _ => greedyRange(frag, None, Some(to))<commit_msg>added non-capturing groups<commit_after>package edu.arizona.sista.matcher

import scala.annotation.tailrec
import scala.util.matching.Regex
import scala.util.parsing.combinator._
import edu.arizona.sista.struct.Interval
import edu.arizona.sista.processors.{Sentence, Document}



object StringMatcherParser extends RegexParsers {
  def parse(input: String): Prog = parseAll(pattern, input) match {
    case Success(result, _) => result
    case failure: NoSuccess => sys.error(failure.msg)
  }

  def int: Parser[Int] = """\d+""".r ^^ { _.toInt }

  def ident: Parser[String] =
    """\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*""".r

  // single- or double-quote delimited string literal
  def quotedLiteral: Parser[String] =
    """'[^\\']*(?:\\.[^\\']*)*'|"[^\\"]*(?:\\.[^\\"]*)*"""".r ^^ {
      case s => """\\(.)""".r.replaceAllIn(s.drop(1).dropRight(1), m => m.group(1))
    }

  def stringLiteral: Parser[String] = ident | quotedLiteral

  // match a perl style "/" delimited regular expression
  // "\" is the escape character, so "\/" becomes "/"
  def regexLiteral: Parser[Regex] = """/[^\\/]*(?:\\.[^\\/]*)*/""".r ^^ {
    case s => s.drop(1).dropRight(1).replaceAll("""\\/""", "/").r
  }

  def exactStringMatcher: Parser[StringMatcher] = stringLiteral ^^ {
    case string => new ExactStringMatcher(string)
  }

  def regexStringMatcher: Parser[StringMatcher] = regexLiteral ^^ {
    case regex => new RegexStringMatcher(regex)
  }

  def stringMatcher: Parser[StringMatcher] = exactStringMatcher | regexStringMatcher

  def fieldConstraint: Parser[TokenConstraint] = ident ~ "=" ~ stringMatcher ^^ {
    case "word" ~ _ ~ matcher => new WordConstraint(matcher)
    case "lemma" ~ _ ~ matcher => new LemmaConstraint(matcher)
    case "tag" ~ _ ~ matcher => new TagConstraint(matcher)
    case "entity" ~ _ ~ matcher => new EntityConstraint(matcher)
    case "chunk" ~ _ ~ matcher => new ChunkConstraint(matcher)
    case _ => sys.error("unrecognized token field")
  }

  def atomicConstraint: Parser[TokenConstraint] = fieldConstraint | "(" ~> disjunctiveConstraint <~ ")"

  def negatedConstraint: Parser[TokenConstraint] = opt("!") ~ atomicConstraint ^^ {
    case None ~ constraint => constraint
    case Some("!") ~ constraint => new NegatedConstraint(constraint)
  }

  def conjunctiveConstraint: Parser[TokenConstraint] = negatedConstraint ~ rep("&" ~> negatedConstraint) ^^ {
    case first ~ rest => (first /: rest) {
      case (lhs, rhs) => new ConjunctiveConstraint(lhs, rhs)
    }
  }

  def disjunctiveConstraint: Parser[TokenConstraint] = conjunctiveConstraint ~ rep("|" ~> conjunctiveConstraint) ^^ {
    case first ~ rest => (first /: rest) {
      case (lhs, rhs) => new DisjunctiveConstraint(lhs, rhs)
    }
  }

  def wordConstraint: Parser[TokenConstraint] = stringMatcher ^^ {
    case matcher => new WordConstraint(matcher)
  }

  def tokenConstraint: Parser[TokenConstraint] = "[" ~> disjunctiveConstraint <~ "]"

  def tokenPattern: Parser[Frag] = (wordConstraint | tokenConstraint) ^^ {
    case constraint => Frag(Match(constraint))
  }

  def capturePattern: Parser[Frag] = "(?<" ~ ident ~ ">" ~ splitPattern ~ ")" ^^ {
    case _ ~ name ~ _ ~ frag ~ _ => capture(name, frag)
  }

  def atomicPattern: Parser[Frag] = tokenPattern | capturePattern | "(" ~> splitPattern <~ ")"

  def repeatedPattern: Parser[Frag] = atomicPattern ~ ("??"|"*?"|"+?"|"?"|"*"|"+") ^^ {
    case pattern ~ "?" => greedyOptional(pattern)
    case pattern ~ "??" => lazyOptional(pattern)
    case pattern ~ "*" => greedyKleene(pattern)
    case pattern ~ "*?" => lazyKleene(pattern)
    case pattern ~ "+" => Frag(pattern, greedyKleene(pattern))
    case pattern ~ "+?" => Frag(pattern, lazyKleene(pattern))
  }

  def rangePattern: Parser[Frag] = atomicPattern ~ "{" ~ int ~ "," ~ int ~ "}" ^^ {
    case frag ~ _ ~ from ~ _ ~ to ~ _ => greedyRange(frag, Some(from), Some(to))
  }

  def fromPattern: Parser[Frag] = atomicPattern ~ "{" ~ int ~ "," ~ "}" ^^ {
    case frag ~ _ ~ from ~ _ ~ _ => greedyRange(frag, Some(from), None)
  }

  def toPattern: Parser[Frag] = atomicPattern ~ "{" ~ "," ~ int ~ "}" ^^ {
    case frag ~ _ ~ _ ~ to ~ _ => greedyRange(frag, None, Some(to))
{'fraction_non_alphanumeric': 0.17869087434304826, 'fraction_numerical': 0.0011944577161968467, 'mean_word_length': 3.885064177362894, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 76, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5830336', 'n_tokens_mistral': 2808, 'n_tokens_neox': 2704, 'n_words': 568}