<commit_before>module Parser(parseEncoreProgram) where
import Text.Parsec
import Text.Parsec.String
import qualified Text.Parsec.Token as P
import Text.Parsec.Language
import Text.Parsec.Expr

import AST

-- Program ::= ClassDecl Program | eps
-- ClassDecl ::= class Name { FieldDecls MethodDecls }
-- FieldDecls ::= Name : Name FieldDecl | eps
-- ParamDecls ::= Name : Name , ParamDecl | eps
-- MethodDecls = def Name ( ParamDecls ) : Name Expr
-- Sequence ::= Expr Seq | eps
-- Seq ::= ; Expr Seq | eps
-- Arguments ::= Expr Args | eps
-- Args ::= , Expr Args | eps
-- Path ::= Name FieldAccess | eps
-- FieldAccess ::= . Name FieldAccess | eps
-- Expr ::= skip
--        | Path = Expr
--        | Path . Name ( Arguments )
--        | let Name :: Name = Expr in Expr
--        | { Sequence }
--        | if Expr then Expr else Expr
--        | get Expr
--        | Path
--        | null
--        | new Name
--        | print Name Name Expr
--        | " String "
--        | Int
--        | ( Expr Op Expr )
-- Op     ::= < | > | == | != | + | - | * | /
-- Name   ::= [a-zA-Z][a-zA-Z0-9]*
-- Int    ::= [0-9]+
-- String ::= ([^"]|\")*

-- Keywords: class def let in if then else get null new print

lexer = 
    P.makeTokenParser $ 
    emptyDef { P.commentStart = "{-",
               P.commentEnd = "-}",
               P.commentLine = "--",
               P.reservedNames = ["class", "def", "skip", "let", "call", "in", "if", "then", "else", "lookup", "get", "null", "new", "print"],
               P.reservedOpNames = [":", "=", "==", "!=", "<", ">", "+", "-", "*", "/"]<commit_msg>Identifiers must now start with a letter<commit_after>module Parser(parseEncoreProgram) where
import Text.Parsec
import Text.Parsec.String
import qualified Text.Parsec.Token as P
import Text.Parsec.Language
import Text.Parsec.Expr

import AST

-- Program ::= ClassDecl Program | eps
-- ClassDecl ::= class Name { FieldDecls MethodDecls }
-- FieldDecls ::= Name : Name FieldDecl | eps
-- ParamDecls ::= Name : Name , ParamDecl | eps
-- MethodDecls = def Name ( ParamDecls ) : Name Expr
-- Sequence ::= Expr Seq | eps
-- Seq ::= ; Expr Seq | eps
-- Arguments ::= Expr Args | eps
-- Args ::= , Expr Args | eps
-- Path ::= Name FieldAccess | eps
-- FieldAccess ::= . Name FieldAccess | eps
-- Expr ::= skip
--        | Path = Expr
--        | Path . Name ( Arguments )
--        | let Name :: Name = Expr in Expr
--        | { Sequence }
--        | if Expr then Expr else Expr
--        | get Expr
--        | Path
--        | null
--        | new Name
--        | print Name Name Expr
--        | " String "
--        | Int
--        | ( Expr Op Expr )
-- Op     ::= < | > | == | != | + | - | * | /
-- Name   ::= [a-zA-Z][a-zA-Z0-9_]*
-- Int    ::= [0-9]+
-- String ::= ([^"]|\")*

-- Keywords: class def let in if then else get null new print

lexer = 
    P.makeTokenParser $ 
    emptyDef { P.commentStart = "{-",
               P.commentEnd = "-}",
               P.commentLine = "--",
               P.identStart = letter,
               P.reservedNames = ["class", "def", "skip", "let", "call", "in", "if", "then", "else", "lookup", "get", "null", "new", "print"],
               P.reservedOpNames = [":", "=", "==", "!=", "<", ">", "+", "-", "*", "/"]
{'fraction_non_alphanumeric': 0.20055452865064696, 'fraction_numerical': 0.0024645717806531116, 'mean_word_length': 1.9625912408759123, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6678239', 'n_tokens_mistral': 1108, 'n_tokens_neox': 1101, 'n_words': 292}