<commit_before>/*
 * Copyright 2013 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package laika.parse

import scala.annotation.tailrec
import scala.collection.mutable.ListBuffer
import scala.util.parsing.combinator.RegexParsers
import scala.util.parsing.input.Reader

/** Base parsers that provide optimized low-level renderers for typical requirements
 *  of text markup parsers. In particular they are meant as an efficient replacement
 *  for scenarios where usually regex parsers are used. In cases where different parsers
 *  need to be tried for relatively short input sequences, regex parsers tend to be less
 *  efficient. Furthermore, these base parsers may also improve readability, as it
 *  allows to combine simple low-level parsers to higher-level parsers based on the
 *  Scala SDK combinator API, instead of producing long regexes which may be hard to read.  
 * 
 *  @author <NAME>
 */
trait MarkupParsers extends RegexParsers {

  
  override def skipWhitespace = false
  

  /** Implicit conversion that allows to pass a single
   *  character to the range-based `anyIn` parser. 
   */
  implicit def charToTraversalble (char: Char): Traversable[Char] = Set(char)
  

  /** Succeeds at the end of a line, including the end of the input.
   *  Produces an empty string as a result and consumes any new line characters.
   */
  def eol: Parser[String] = Parser { in =>
      if (in.atEnd) Success("", in) 
      else if (in.first == '\n') Success("", in.rest)
      else if (in.first == '\r' && in.source.length > in.offset + 1 && in.source.charAt(in.offset + 1) == '\n') Success("", in.drop(2))
      else Failure("Not at end of line", in)
  }  
  
  /** Succeeds at the end of the input.
   */
  def eof: Parser[String] = Parser { in =>
      if (in.atEnd) Success("", in) 
      else Failure("Not at end of input", in)
  }  
  
  /** Parses horizontal whitespace (space and tab).
   *  Always succeeds, consuming all whitespace found.
   */
  def ws = anyOf(' ','\t')
  
  
  /** API for specifying further constraints on the parsers provided by this base trait.
   * 
   *  For reading 3 or more `'*'` or `'+'`characters for example the constraint could
   *  be specified as follows:
   *  
   *  {{{
   *  anyOf('*','+') min 3
   *  }}}
   */
  class TextParser private[MarkupParsers] (newParser:        (Int, Int, Boolean, Boolean, Char => Boolean) => Parser[(String,Boolean)], 
                                           minChar:          Int = 0, 
                                           maxChar:          Int = 0, 
                                           mustFailAtEOF:    Boolean = false,
                                           consumeLastChar: Boolean = false,
                                           isStopChar:       Char => Boolean = c => false) extends Parser[String] {
    
    private val parser = newParser(minChar, maxChar, mustFailAtEOF, consumeLastChar, isStopChar)
    
    /** Creates and returns a new parser that fails if it does not consume the specified minimum number
     *  of characters. It may still consume more characters in case of further matches. 
     */
    def min (count: Int) = new TextParser(newParser, count, maxChar, mustFailAtEOF, consumeLastChar, isStopChar)
    
    /** Creates and returns a new parser that consumes at most the specified maximum number of characters. 
     *  Always succeeds, unless a minimum number of matches is also specified.
     */
    def max (count: Int) = new TextParser(newParser, minChar, count, mustFailAtEOF, consumeLastChar, isStopChar)
    
    /** Creates and returns a new parser that consumes exactly the specified numer of characters.
     *  Fails if there are less matches, but succeeds in case there are more matches, simply ignoring them.
     *  Calling `take 3` for example is equivalent to calling `min 3 max 3`.
     */
    def take (count: Int) = new TextParser(newParser, count, count, mustFailAtEOF, consumeLastChar, isStopChar)
    
    private[parse] def failAtEOF = new TextParser(newParser, minChar, maxChar, true, consumeLastChar, isStopChar)
  
    private[parse] def stopChars (chars: Char*) = new TextParser(newParser, minChar, maxChar, mustFailAtEOF, consumeLastChar, charLookupFor(chars:_*))
    
    private[parse] def consumeLastChar = new TextParser(newParser, minChar, maxChar, mustFailAtEOF, true, isStopChar)
    
    private[parse] def applyInternal (in: Input) = parser(in)
    
    def apply (in: Input) = parser(in) match {
      case Success((result,_), next) => Success(result, next)
      case ns: NoSuccess => ns
    }
  }
  
  
  /** Returns an optimized, Array-based lookup function 
   *  for the specified characters.
   */
  protected def optimizedCharLookup (chars: Char*) = {
    val max = chars.max
    val lookup = new Array[Int](max + 1)
    
    for (c <- chars) lookup(c) = 1
    
    c:Char => c <= max && lookup(c) == 1
  }
  
  /** Returns an optimized, Array-based lookup function 
   *  for the specified ranges of characters.
   */
  protected def optimizedRangeLookup (ranges: Traversable[Char]*) = {
    val max = ranges map (_.max) max
    val lookup = new Array[Int](max + 1)
    
    for (r <- ranges; c <- r) lookup(c) = 1
    
    c:Char => c <= max && lookup(c) == 1
  }
  
  private def charLookupFor (chars: Char*): Char => Boolean = {
    chars.length match {
      case 0 => c => false
      case 1 => val c = chars(0); _ == c
      case 2 => val c1 = chars(0); val c2 = chars(1); c => c == c1 || c == c2
      case _ => optimizedCharLookup(chars:_*)
    } 
  }
  
  /** Consumes any kind of input, always succeeds.
   *  This parser would consume the entire input unless a `max` constraint
   *  is specified.
   */
  val any = {
    anyWhile(c=>true)
  }
  
  /** Consumes any number of consecutive occurrences of the specified characters.
   *  Always succeeds unless a minimum number of required matches is specified.
   */
  def anyOf (chars: Char*) = {
    anyWhile(charLookupFor(chars:_*))
  }
  
  /** Consumes any number of consecutive characters that are not one of the specified characters.
   *  Always succeeds unless a minimum number of required matches is specified.
   */
  def anyBut (chars: Char*) = {
    val p: Char => Boolean = chars.length match {
      case 0 => c => true
      case 1 => val c = chars(0); _ != c
      case 2 => val c1 = chars(0); val c2 = chars(1); c => c != c1 && c != c2
      case _ => val lookup = optimizedCharLookup(chars:_*); !lookup(_)
    } 
    anyWhile(p)
  }
  
  /** Consumes any number of consecutive characters that are not one of the specified characters.
   *  Always succeeds unless a minimum number of required matches is specified.
   *  
   *  This parser is identical to the `anyBut` parser except for two differences: this parser fails
   *  if it reaches the end of the input without seeing any of the specified
   *  characters and it also consumes this final character, without adding it
   *  to the result. This parser is usually used when a construct like a span
   *  enclosed between two characters needs to be parsed.
   */
  def anyUntil (chars: Char*) = anyBut(chars:_*).failAtEOF.consumeLastChar
  
  /** Consumes any number of consecutive characters that are in one of the specified character ranges.
   *  Always succeeds unless a minimum number of required matches is specified.
   */
  def anyIn (ranges: Traversable[Char]*) = {
    val p: Char => Boolean = {
      if (ranges.isEmpty) c => false
      else optimizedRangeLookup(ranges:_*)
    }
    anyWhile(p)
  }
  
  /** Consumes any number of consecutive characters for which satisfy the specified predicate.
   *  Always succeeds unless a minimum number of required matches is specified.
   */<commit_msg>Fix Scaladoc in MarkupParsers.anyUntil and anyWhile<commit_after>/*
 * Copyright 2013 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package laika.parse

import scala.annotation.tailrec
import scala.collection.mutable.ListBuffer
import scala.util.parsing.combinator.RegexParsers
import scala.util.parsing.input.Reader

/** Base parsers that provide optimized low-level renderers for typical requirements
 *  of text markup parsers. In particular they are meant as an efficient replacement
 *  for scenarios where usually regex parsers are used. In cases where different parsers
 *  need to be tried for relatively short input sequences, regex parsers tend to be less
 *  efficient. Furthermore, these base parsers may also improve readability, as it
 *  allows to combine simple low-level parsers to higher-level parsers based on the
 *  Scala SDK combinator API, instead of producing long regexes which may be hard to read.  
 * 
 *  @author <NAME>
 */
trait MarkupParsers extends RegexParsers {

  
  override def skipWhitespace = false
  

  /** Implicit conversion that allows to pass a single
   *  character to the range-based `anyIn` parser. 
   */
  implicit def charToTraversalble (char: Char): Traversable[Char] = Set(char)
  

  /** Succeeds at the end of a line, including the end of the input.
   *  Produces an empty string as a result and consumes any new line characters.
   */
  def eol: Parser[String] = Parser { in =>
      if (in.atEnd) Success("", in) 
      else if (in.first == '\n') Success("", in.rest)
      else if (in.first == '\r' && in.source.length > in.offset + 1 && in.source.charAt(in.offset + 1) == '\n') Success("", in.drop(2))
      else Failure("Not at end of line", in)
  }  
  
  /** Succeeds at the end of the input.
   */
  def eof: Parser[String] = Parser { in =>
      if (in.atEnd) Success("", in) 
      else Failure("Not at end of input", in)
  }  
  
  /** Parses horizontal whitespace (space and tab).
   *  Always succeeds, consuming all whitespace found.
   */
  def ws = anyOf(' ','\t')
  
  
  /** API for specifying further constraints on the parsers provided by this base trait.
   * 
   *  For reading 3 or more `'*'` or `'+'`characters for example the constraint could
   *  be specified as follows:
   *  
   *  {{{
   *  anyOf('*','+') min 3
   *  }}}
   */
  class TextParser private[MarkupParsers] (newParser:        (Int, Int, Boolean, Boolean, Char => Boolean) => Parser[(String,Boolean)], 
                                           minChar:          Int = 0, 
                                           maxChar:          Int = 0, 
                                           mustFailAtEOF:    Boolean = false,
                                           consumeLastChar: Boolean = false,
                                           isStopChar:       Char => Boolean = c => false) extends Parser[String] {
    
    private val parser = newParser(minChar, maxChar, mustFailAtEOF, consumeLastChar, isStopChar)
    
    /** Creates and returns a new parser that fails if it does not consume the specified minimum number
     *  of characters. It may still consume more characters in case of further matches. 
     */
    def min (count: Int) = new TextParser(newParser, count, maxChar, mustFailAtEOF, consumeLastChar, isStopChar)
    
    /** Creates and returns a new parser that consumes at most the specified maximum number of characters. 
     *  Always succeeds, unless a minimum number of matches is also specified.
     */
    def max (count: Int) = new TextParser(newParser, minChar, count, mustFailAtEOF, consumeLastChar, isStopChar)
    
    /** Creates and returns a new parser that consumes exactly the specified numer of characters.
     *  Fails if there are less matches, but succeeds in case there are more matches, simply ignoring them.
     *  Calling `take 3` for example is equivalent to calling `min 3 max 3`.
     */
    def take (count: Int) = new TextParser(newParser, count, count, mustFailAtEOF, consumeLastChar, isStopChar)
    
    private[parse] def failAtEOF = new TextParser(newParser, minChar, maxChar, true, consumeLastChar, isStopChar)
  
    private[parse] def stopChars (chars: Char*) = new TextParser(newParser, minChar, maxChar, mustFailAtEOF, consumeLastChar, charLookupFor(chars:_*))
    
    private[parse] def consumeLastChar = new TextParser(newParser, minChar, maxChar, mustFailAtEOF, true, isStopChar)
    
    private[parse] def applyInternal (in: Input) = parser(in)
    
    def apply (in: Input) = parser(in) match {
      case Success((result,_), next) => Success(result, next)
      case ns: NoSuccess => ns
    }
  }
  
  
  /** Returns an optimized, Array-based lookup function 
   *  for the specified characters.
   */
  protected def optimizedCharLookup (chars: Char*) = {
    val max = chars.max
    val lookup = new Array[Int](max + 1)
    
    for (c <- chars) lookup(c) = 1
    
    c:Char => c <= max && lookup(c) == 1
  }
  
  /** Returns an optimized, Array-based lookup function 
   *  for the specified ranges of characters.
   */
  protected def optimizedRangeLookup (ranges: Traversable[Char]*) = {
    val max = ranges map (_.max) max
    val lookup = new Array[Int](max + 1)
    
    for (r <- ranges; c <- r) lookup(c) = 1
    
    c:Char => c <= max && lookup(c) == 1
  }
  
  private def charLookupFor (chars: Char*): Char => Boolean = {
    chars.length match {
      case 0 => c => false
      case 1 => val c = chars(0); _ == c
      case 2 => val c1 = chars(0); val c2 = chars(1); c => c == c1 || c == c2
      case _ => optimizedCharLookup(chars:_*)
    } 
  }
  
  /** Consumes any kind of input, always succeeds.
   *  This parser would consume the entire input unless a `max` constraint
   *  is specified.
   */
  val any = {
    anyWhile(c=>true)
  }
  
  /** Consumes any number of consecutive occurrences of the specified characters.
   *  Always succeeds unless a minimum number of required matches is specified.
   */
  def anyOf (chars: Char*) = {
    anyWhile(charLookupFor(chars:_*))
  }
  
  /** Consumes any number of consecutive characters that are not one of the specified characters.
   *  Always succeeds unless a minimum number of required matches is specified.
   */
  def anyBut (chars: Char*) = {
    val p: Char => Boolean = chars.length match {
      case 0 => c => true
      case 1 => val c = chars(0); _ != c
      case 2 => val c1 = chars(0); val c2 = chars(1); c => c != c1 && c != c2
      case _ => val lookup = optimizedCharLookup(chars:_*); !lookup(_)
    } 
    anyWhile(p)
  }
  
  /** Consumes any number of consecutive characters that are not one of the specified characters.
   *  
   *  This parser is identical to the `anyBut` parser except for two differences: this parser fails
   *  if it reaches the end of the input without seeing any of the specified
   *  characters and it also consumes this final character, without adding it
   *  to the result. This parser is usually used when a construct like a span
   *  enclosed between two characters needs to be parsed.
   */
  def anyUntil (chars: Char*) = anyBut(chars:_*).failAtEOF.consumeLastChar
  
  /** Consumes any number of consecutive characters that are in one of the specified character ranges.
   *  Always succeeds unless a minimum number of required matches is specified.
   */
  def anyIn (ranges: Traversable[Char]*) = {
    val p: Char => Boolean = {
      if (ranges.isEmpty) c => false
      else optimizedRangeLookup(ranges:_*)
    }
    anyWhile(p)
  }
  
  /** Consumes any number of consecutive characters which satisfy the specified predicate.
   *  Always succeeds unless a minimum number of required matches is specified.
   */
{'fraction_non_alphanumeric': 0.09385603490063014, 'fraction_numerical': 0.005332040717401842, 'mean_word_length': 2.75882486905033, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4144856', 'n_tokens_mistral': 4757, 'n_tokens_neox': 4576, 'n_words': 1889}