<commit_before>module Jana.Parser where

import Text.Parsec
import Text.Parsec.String (Parser)
import qualified Text.Parsec.Token as T

import Jana.Ast


janusDef :: T.LanguageDef st
janusDef = T.LanguageDef
    { T.commentStart    = ""
    , T.commentEnd      = ""
    , T.commentLine     = "#"
    , T.nestedComments  = True
    , T.identStart      = letter <|> char '_'
    , T.identLetter     = alphaNum <|> oneOf "_-"
    , T.opStart         = T.opLetter janusDef
    , T.opLetter        = oneOf "+-*/%&|<=>^"
    , T.reservedOpNames = []
    , T.reservedNames   = []
    , T.caseSensitive   = True
    }

varType :: Parser Type
varType =
        (reserved "int" >> return Int)
    <|> (reserved "stack" >> return Stack)

param :: Parser (Type, Ident)
param = do
    parType <- varType
    parName <- ident
    return (parType, parName)

paramList :: Parser [(Type, Ident)]
paramList = commaSep param

proc :: Parser Proc
proc = do
    reserved "procedure"
    name <- ident
    ps   <- parens paramList
    return Proc { procname = name, params = ps, body = [] }

parserTest =
    case (parse proc "" "procedure foo(int x)") of
        Left err -> error $ show err
        Right p  -> print p


-- Lexer
lexer = T.makeTokenParser janusDef

ident    = T.identifier lexer
reserved = T.reserved lexer
parens   = T.parens lexer
commaSep = T.commaSep lexer<commit_msg>Added new parserfile with some training/junk<commit_after>module Jana.Parser where

import Jana.Ast
import Text.Parsec
import Text.Parsec.String
import Text.Parsec.Expr
import Text.Parsec.Token
import Text.Parsec.Language

def = emptyDef{
    commentStart     = "(*",
    commentEnd       = "*)",
    identStart       = letter,
    identLetter      = alphaNum,
    opStart          = oneOf "+-^=",
    opLetter         = oneOf "+-^=",
    reservedOpNames  = ["+", "-", "^", "="],
    reservedNames    = ["procedure",
                        "int", "stack",
                        "if", "then", "else", "fi",
                        "from", "do", "loop", "until",
                        "push", "pop",
                        "local", "delocal",
                        "call", "uncall",
                        "skip",
                        "empty", "top",
                        "nil"]
  }

TokenParser{ parens     = m_parens
           , identifier = m_identifier
           , reservedOp = m_reservedOp
           , reserved   = m_reserved
           , semiSep1   = m_semiSep1
           , whiteSpace = m_whiteSpace } = makeTokenParser def


main :: IO ()
main = play("hey")
{'fraction_non_alphanumeric': 0.12823529411764706, 'fraction_numerical': 0.000784313725490196, 'mean_word_length': 1.9021615472127418, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3904098', 'n_tokens_mistral': 830, 'n_tokens_neox': 780, 'n_words': 197}