<commit_before>package edu.colorado.plv.cuanto.jsy.common

import edu.colorado.plv.cuanto.jsy._
import edu.colorado.plv.cuanto.parsing.RichParsers

import scala.util.parsing.combinator.RegexParsers

/** Parser components that handle unary and binary operators.
  *
  * This trait implements standard non-terminals for an expression language
  * with unary and binary operators.
  *
  * ==Abstract Syntax==
  *
  * This trait implements a parser for the following abstract syntax
  * consisting of atoms, unary, and binary expressions.
  *
  * ''expr'' ::= ''atom'' | ''uop'' ''expr'' | ''expr'' ''bop'' ''expr''
  *
  * ==Concrete Syntax==
  *
  * Specifically, it implements a parser given a specification for the atoms,
  * unary, and binary operators for the following concrete syntax:
  *
  * ''binary,,1,,'' ::= ''binary,,2,,'' { ''bop,,1,,'' ''binary,,2,,'' }
  *
  *  ...
  *
  * ''binary,,n,,'' ::= ''unary'' { ''bop,,n,,'' ''unary'' }
  *
  * ''unary'' ::= ''uop'' ''unary'' | ''atom''
  *
  * ''atom'' ::= ''opatom'' | `(` ''expr'' `)`
  *
  * The { α } is in the meta-language to indicate 0-or-more α's (i.e., EBNF).
  *
  * The ''expr'', ''opatom'', and ''uop'' symbols are specified by defining
  * [[scala.util.parsing.combinator.Parsers#Parser]]s.
  *
  * The ''bop'' symbols is specified by defining `bop`, which should be a
  * list of list of concrete-abstract syntax pairs. The outer list specifies
  * the precedence levels from lowest to highest, while the inner list are
  * the operators at the same level. All operators are treated as
  * left-associative.
  *
  * @example {{{
  *
  *   trait ArithmeticParser extends OpParserLike with JavaTokenParsers {
  *     override def opatom: Parser[Expr] =
  *       positioned {
  *         floatingPointNumber ^^ (s => N(s.toDouble))
  *       }
  *     override def uop: Parser[Uop] =
  *       "-" ^^ { _ => Neg }
  *     override def bop: OpPrecedence = Seq(
  *       /* lowest */
  *       Seq("+" -> Plus, "-" -> Minus),
  *       Seq("*" -> Times, "/" -> Div)
  *       /* highest */
  *     )
  *   }
  *
  * }}}
  *
  * @author Bor-Yuh <NAME>
  */
trait OpParserLike extends RegexParsers with RichParsers {
  /** Define expressions. */
  def expr: Parser[Expr]

  /** Define atoms. */
  def opatom: Parser[Expr]

  /** Define unary operators. */
  def uop: Parser[Uop]

  /** Type alias for the list defining the precedence of binary operators.
    *
    * @see [[bop]] for defining the precedence of binary operators.
    */
  type OpPrecedence = Seq[Seq[(String,Bop)]]

  /** Define precedence of left-associative binary operators.
    *
    * Specified from lowest to highest, consisting of pairs of
    * concrete-abstract syntax.
    */
  val bop: OpPrecedence

  /** Parser for binary expressions. */
  def binary: Parser[Expr] = {
    def binaryOps(ops: OpPrecedence): Parser[Expr] = {
      def binaryCase(opsyn: (String, Bop)): Parser[(Expr, Expr) => Expr] = {
        val (csyn, asyn) = opsyn
        withpos(csyn) ^^ { case (pos, _) => (e1: Expr, e2: Expr) => Binary(asyn, e1, e2) setPos pos }
      }
      def level(ops: Seq[(String, Bop)]): Parser[(Expr, Expr) => Expr] = {
        val op1 :: t = ops
        t.foldLeft(binaryCase(op1)) { (acc, op) => acc | binaryCase(op) }
      }
      ops.foldRight(unary) { (lops, acc) => acc * level(lops) }
    }
    binaryOps(bop)
  }

  /** Parser for unary expressions. */
  def unary: Parser[Expr] =
    positioned {
      uop ~ unary ^^ { case op ~ e => Unary(op, e) }
    } |
    atom

  /** Parser for atoms. */
  def atom: Parser[Expr] =
    opatom |
    "(" ~> expr <~ ")" |<commit_msg>Extend OpParserLike with parsing blocks<commit_after>package edu.colorado.plv.cuanto.jsy.common

import edu.colorado.plv.cuanto.jsy._
import edu.colorado.plv.cuanto.parsing.RichParsers

import scala.util.parsing.combinator.RegexParsers

/** Parser components that handle unary and binary operators.
  *
  * This trait implements standard non-terminals for an expression language
  * with unary and binary operators.
  *
  * ==Abstract Syntax==
  *
  * This trait implements a parser for the following abstract syntax
  * consisting of atoms, unary, and binary expressions.
  *
  *   - ''expr'' ::= ''atom'' | ''uop'' ''expr'' | ''expr'' ''bop'' ''expr''
  *
  * ==Concrete Syntax==
  *
  * Specifically, it implements a parser given a specification for the atoms,
  * unary, and binary operators for the following concrete syntax:
  *
  *   - ''binary,,1,,'' ::= ''binary,,2,,'' { ''bop,,1,,'' ''binary,,2,,'' }
  *   - ...
  *   - ''binary,,n,,'' ::= ''unary'' { ''bop,,n,,'' ''unary'' }
  *   - ''unary'' ::= ''uop'' ''unary'' | ''atom''
  *   - ''atom'' ::= ''opatom'' | `(` ''expr'' `)` | `{` ''expr'' `}`
  *
  * The { α } is in the meta-language to indicate 0-or-more α's (i.e., EBNF).
  *
  * The ''expr'', ''opatom'', and ''uop'' symbols are specified by defining
  * [[scala.util.parsing.combinator.Parsers#Parser]]s.
  *
  * The ''bop'' symbols is specified by defining `bop`, which should be a
  * list of list of concrete-abstract syntax pairs. The outer list specifies
  * the precedence levels from lowest to highest, while the inner list are
  * the operators at the same level. All operators are treated as
  * left-associative.
  *
  * @example {{{
  *
  *   trait ArithmeticParser extends OpParserLike with JavaTokenParsers {
  *     override def opatom: Parser[Expr] =
  *       positioned {
  *         floatingPointNumber ^^ (s => N(s.toDouble))
  *       }
  *     override def uop: Parser[Uop] =
  *       "-" ^^ { _ => Neg }
  *     override def bop: OpPrecedence = Seq(
  *       /* lowest */
  *       Seq("+" -> Plus, "-" -> Minus),
  *       Seq("*" -> Times, "/" -> Div)
  *       /* highest */
  *     )
  *   }
  *
  * }}}
  *
  * @author <NAME>
  */
trait OpParserLike extends RegexParsers with RichParsers {
  /** Parameter: define expressions. */
  def expr: Parser[Expr]

  /** Parameter: define atoms. */
  def opatom: Parser[Expr]

  /** Parameter: define unary operators. */
  def uop: Parser[Uop]

  /** Type alias for the list defining the precedence of binary operators.
    *
    * @see [[bop]] for defining the precedence of binary operators.
    */
  type OpPrecedence = Seq[Seq[(String,Bop)]]

  /** Parameter: define precedence of left-associative binary operators.
    *
    * Specified from lowest to highest, consisting of pairs of
    * concrete-abstract syntax.
    */
  val bop: OpPrecedence

  /** Yields a parser for binary expressions with an instantiation for `bop`.
    *
    * @see bop
    */
  def binary: Parser[Expr] = {
    def binaryOps(ops: OpPrecedence): Parser[Expr] = {
      def binaryCase(opsyn: (String, Bop)): Parser[(Expr, Expr) => Expr] = {
        val (csyn, asyn) = opsyn
        withpos(csyn) ^^ { case (pos, _) => (e1: Expr, e2: Expr) => Binary(asyn, e1, e2) setPos pos }
      }
      def level(ops: Seq[(String, Bop)]): Parser[(Expr, Expr) => Expr] = {
        val op1 :: t = ops
        t.foldLeft(binaryCase(op1)) { (acc, op) => acc | binaryCase(op) }
      }
      ops.foldRight(unary) { (lops, acc) => acc * level(lops) }
    }
    binaryOps(bop)
  }

  /** Yields a parser for unary expressions with an instantiation for `uop`
    *
    * @see uop
    */
  def unary: Parser[Expr] =
    positioned {
      uop ~ unary ^^ { case op ~ e => Unary(op, e) }
    } |
    atom

  /** Parse parenthesized expressions and delegates to `opatom`.
    *
    * @see opatom
    */
  def atom: Parser[Expr] =
    opatom |
    "(" ~> expr <~ ")" |         // parentheses
    "{" ~> expr <~ "}" |         // blocks (just treat them as parentheses)
{'fraction_non_alphanumeric': 0.1634602800680539, 'fraction_numerical': 0.0028792042926318543, 'mean_word_length': 2.7169260700389106, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5452456', 'n_tokens_mistral': 2668, 'n_tokens_neox': 2608, 'n_words': 728}