<commit_before>package ducttape.syntax

import java.io.File
import java.math.BigDecimal
import java.math.BigInteger

import org.apache.commons.lang3.StringEscapeUtils;

import scala.util.parsing.combinator.Parsers
import scala.util.parsing.combinator.RegexParsers
import scala.util.parsing.input.CharArrayReader
import scala.util.parsing.input.Position
import scala.util.parsing.input.Positional
import scala.util.matching.Regex

import ducttape.syntax.GrammarParser._
import ducttape.syntax.AbstractSyntaxTree._


object Grammar {
  
  
  /** End of line characters, including end of file. */
  val eol: Parser[String] = literal("\r\n") | literal("\n") | regex("""\z""".r) | literal(CharArrayReader.EofCh.toString) 
 
  /** Non-end of line white space characters */
  val space: Parser[String] = regex("""[ \t]+""".r)
  
  /** One or more whitespace characters */
  val whitespace: Parser[String] = regex("""\s+""".r)

  

  object Keyword {
    
    private def keyword(word:String):Parser[String] = {
      (
          literal(word) |
          failure("""The keyword """"+word+"""" is required but missing.""")
      ) <~ 
      (
          space |
          err("""At least one space or tab must immediately follow the """"+word+"""" keyword.""")
      )
    }
       
    val task:Parser[String] = keyword("task")
    val group:Parser[String] = keyword("group")
    val func:Parser[String] = keyword("func")
    val summmary:Parser[String] = keyword("summary")
    val of:Parser[String] = keyword("of")
    val action:Parser[String] = keyword("action")
    val submitter:Parser[String] = keyword("submitter")
    val versioner:Parser[String] = keyword("versioner")
    val packageKeyword:Parser[String] = keyword("package")
    val branchpoint:Parser[String] = keyword("branchpoint")
    val baseline:Parser[String] = keyword("baseline")
    val branch:Parser[String] = keyword("branch")
    val config:Parser[String] = keyword("config")
  }
  
  /** One line of comments */
  val comment: Parser[String] = {
      opt(space) ~>
      (literal("//") | literal("#")) ~>
      regex("""[^\r\n]*""".r) <~
      eol
  }
  
  /** One or more lines of comments */
  val comments: Parser[Comments] = {
    repsep(comment,opt(whitespace)) <~ opt(whitespace)
  } ^^ {
    case list:List[String] => {
      if (list.isEmpty) {
        new Comments(None)
      } else {
        new Comments(Some(list.mkString("\n")))
      } 
    }
  }

  
  /** A signed, arbitrary precision number. */
  val number: Parser[BigDecimal] = 
      ( // Recognize a number with at least one digit left of the decimal
          // and optionally, one or more digits to the right of the decimal
          regex("""[+-]?\d+(\.\d+)?([eE][-+]?\d+)?""".r)  |
          // Do NOT recognize a number with no digits left of the decimal
          (regex("""[+-]?\.\d+([eE][-+]?\d+)?""".r)~!err("A number must have at least one digit left of the decimal point.") )<commit_msg>Added keywords reach, via, and plan.<commit_after>package ducttape.syntax

import java.io.File
import java.math.BigDecimal
import java.math.BigInteger

import org.apache.commons.lang3.StringEscapeUtils;

import scala.util.parsing.combinator.Parsers
import scala.util.parsing.combinator.RegexParsers
import scala.util.parsing.input.CharArrayReader
import scala.util.parsing.input.Position
import scala.util.parsing.input.Positional
import scala.util.matching.Regex

import ducttape.syntax.GrammarParser._
import ducttape.syntax.AbstractSyntaxTree._


object Grammar {
  
  
  /** End of line characters, including end of file. */
  val eol: Parser[String] = literal("\r\n") | literal("\n") | regex("""\z""".r) | literal(CharArrayReader.EofCh.toString) 
 
  /** Non-end of line white space characters */
  val space: Parser[String] = regex("""[ \t]+""".r)
  
  /** One or more whitespace characters */
  val whitespace: Parser[String] = regex("""\s+""".r)

  

  object Keyword {
    
    private def keyword(word:String):Parser[String] = {
      (
          literal(word) |
          failure("""The keyword """"+word+"""" is required but missing.""")
      ) <~ 
      (
          space |
          err("""At least one space or tab must immediately follow the """"+word+"""" keyword.""")
      )
    }
       
    val task:Parser[String] = keyword("task")
    val group:Parser[String] = keyword("group")
    val func:Parser[String] = keyword("func")
    val summmary:Parser[String] = keyword("summary")
    val of:Parser[String] = keyword("of")
    val action:Parser[String] = keyword("action")
    val submitter:Parser[String] = keyword("submitter")
    val versioner:Parser[String] = keyword("versioner")
    val packageKeyword:Parser[String] = keyword("package")
    val branchpoint:Parser[String] = keyword("branchpoint")
    val baseline:Parser[String] = keyword("baseline")
    val branch:Parser[String] = keyword("branch")
    val config:Parser[String] = keyword("config")
    val reach:Parser[String] = keyword("reach")
    val via:Parser[String] = keyword("via")
    val plan:Parser[String] = keyword("plan")
  }
  
  /** One line of comments */
  val comment: Parser[String] = {
      opt(space) ~>
      (literal("//") | literal("#")) ~>
      regex("""[^\r\n]*""".r) <~
      eol
  }
  
  /** One or more lines of comments */
  val comments: Parser[Comments] = {
    repsep(comment,opt(whitespace)) <~ opt(whitespace)
  } ^^ {
    case list:List[String] => {
      if (list.isEmpty) {
        new Comments(None)
      } else {
        new Comments(Some(list.mkString("\n")))
      } 
    }
  }

  
  /** A signed, arbitrary precision number. */
  val number: Parser[BigDecimal] = 
      ( // Recognize a number with at least one digit left of the decimal
          // and optionally, one or more digits to the right of the decimal
          regex("""[+-]?\d+(\.\d+)?([eE][-+]?\d+)?""".r)  |
          // Do NOT recognize a number with no digits left of the decimal
          (regex("""[+-]?\.\d+([eE][-+]?\d+)?""".r)~!err("A number must have at least one digit left of the decimal point.") )
{'fraction_non_alphanumeric': 0.16399139500248222, 'fraction_numerical': 0.00033096144299189144, 'mean_word_length': 3.1596696490020646, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2130675', 'n_tokens_mistral': 1962, 'n_tokens_neox': 1911, 'n_words': 450}