<commit_before>{-# LANGUAGE OverloadedStrings #-}
module Language.Hasquito.Parser where
import           Control.Applicative
import           Data.Attoparsec.Text
import qualified Data.Text as T
import qualified Data.Text.IO as TIO
import           Language.Hasquito.Syntax
import           Language.Hasquito.Util

name :: Parser Name
name = Name . T.pack <$> many1 letter

tparen :: Parser Ty
tparen = char '(' *> ty <* char ')'

tnum :: Parser Ty
tnum = string "Num" *> return TNum

tvar :: Parser Ty
tvar = TVar <$> name

ty :: Parser Ty
ty = do
  t <- nonrec
  try (TArr t <$> (string "->" *> ty)) <|> return t
  where nonrec = skipSpace *> (tparen <|> tnum <|> tvar) <* skipSpace

num :: Parser Exp
num = Num <$> signed decimal

var :: Parser Exp
var = Var <$> name

app :: Parser Exp
app = do
  l <- expr
  char ' ' *> skipSpace
  r <- expr
  return (App l r)

bindings :: Parser [(Name, Ty)]
bindings = many1 (skipSpace *> binding)
  where binding = do
          char '(' *> skipSpace 
          n <- name
          skipSpace *> char ':'
          t <- ty
          return (n, t)

lam :: Parser Exp
lam = do
  char '('
  vars <- bindings
  skipSpace *> string " -> " *> skipSpace
  body <- expr
  char ')'
  return (Lam vars body)

eparen :: Parser Exp
eparen = char '(' *> expr <* skipSpace <* char ')'

expr :: Parser Exp
expr = skipSpace *> (eparen <|> try app <|> try lam <|> var <|> num)

def :: Parser (Def ())
def = do
  skipSpace
  nm <- name
  skipSpace *> char ':' *> skipSpace
  t <- ty
  skipSpace *> char '=' *> skipSpace
  ex <- expr
  skipSpace <* char ';'
  return (Def t nm ex ())

file :: T.Text -> Either String [Def ()]
file = parseOnly (many def <* skipSpace)

parseFile :: FilePath -> IO (Either Error [Def ()])
parseFile path = mapL (ParseError . T.pack)  . file <$> (TIO.readFile path)
  where mapL f (Left a)  = Left (f a)
        mapL _ (Right b) = Right b
<commit_msg>Parser is now somewhat working<commit_after>{-# LANGUAGE OverloadedStrings #-}
module Language.Hasquito.Parser where
import           Control.Applicative
import           Data.Attoparsec.Text
import qualified Data.Text as T
import qualified Data.Text.IO as TIO
import           Language.Hasquito.Syntax
import           Language.Hasquito.Util
import           Data.List (foldl1')

name :: Parser Name
name = Name . T.pack <$> many1 letter

tparen :: Parser Ty
tparen = char '(' *> ty <* char ')'

tnum :: Parser Ty
tnum = string "Num" *> return TNum

tvar :: Parser Ty
tvar = TVar <$> name

ty :: Parser Ty
ty = do
  t <- nonrec
  try (TArr t <$> (string "->" *> ty)) <|> return t
  where nonrec = skipSpace *> (tparen <|> tnum <|> tvar) <* skipSpace

num :: Parser Exp
num = Num <$> signed decimal

var :: Parser Exp
var = Var <$> name

bindings :: Parser [(Name, Ty)]
bindings = many1 (skipSpace *> binding)
  where binding = do
          char '(' *> skipSpace 
          n <- name
          skipSpace *> char ':'
          t <- ty
          skipSpace <* char ')'
          return (n, t)

lam :: Parser Exp
lam = do
  string "fun "
  vars <- bindings
  skipSpace *> string "->"
  body <- expr
  return (Lam vars body)

eparen :: Parser Exp
eparen = char '(' *> expr <* skipSpace <* char ')'

expr :: Parser Exp
expr = foldl1' App <$> many1 nonrec
  where nonrec = skipSpace *> (try lam <|> eparen <|> var <|> num)

def :: Parser (Def ())
def = do
  skipSpace
  nm <- name
  skipSpace *> char ':' *> skipSpace
  t <- ty
  skipSpace *> char '=' *> skipSpace
  ex <- expr
  skipSpace <* char ';'
  return (Def t nm ex ())

file :: T.Text -> Either String [Def ()]
file = parseOnly (many def <* skipSpace)

parseFile :: FilePath -> IO (Either Error [Def ()])
parseFile path = mapL (ParseError . T.pack)  . file <$> (TIO.readFile path)
  where mapL f (Left a)  = Left (f a)
        mapL _ (Right b) = Right b

{'fraction_non_alphanumeric': 0.14548319327731093, 'fraction_numerical': 0.001838235294117647, 'mean_word_length': 2.6070075757575757, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4988400', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1324, 'n_words': 445}