<commit_before>{
module Language.Dash.Parser.Parser where

import Language.Dash.Parser.Lexer
import Language.Dash.IR.Ast
import Language.Dash.BuiltIn.BuiltInDefinitions

import Data.List (sortBy)

}

%name       parse
%tokentype  { Token }
%error      { parseError }

%token
  eol       { TEOL }
  '('       { TOpen_Par }
  ')'       { TClose_Par }
  '['       { TOpen_Bracket }
  ']'       { TClose_Bracket }
  '{'       { TOpen_Brace }
  '}'       { TClose_Brace }
  module    { TModule }
  if        { TIf }
  then      { TThen }
  else      { TElse }
  '='       { TDefine }
  symbol    { TSymbol $$ }
  id        { TId $$ }
  ns        { TNamespace $$ }
  -- TODO qid       { TQId $$ }
  string    { TString $$ }
  int       { TInt $$ }
  match     { TMatch }
  do        { TDo }
  '->'      { TArrow_R }
  '<-'      { TArrow_L }
  with      { TWith }
  -- begin     { TBegin }
  end       { TEnd }
  '+'       { TOperator "+" }
  '-'       { TOperator "-" }
  '/'       { TOperator "/" }
  '*'       { TOperator "*" }
  '=='      { TOperator "==" }
  '<'       { TOperator "<" }
  '>'       { TOperator ">" }
  '<='      { TOperator "<=" }
  '>='      { TOperator ">=" }
  '^+'      { TOperator "^+" }
  '++'      { TOperator "++" }
  '||'      { TOperator "||" }
  '&&'      { TOperator "&&" }
  '!'       { TOperator "!" }
  operator  { TOperator $$ }
  '_'       { TUnderscore }
  ','       { TComma }
  '|'       { TVBar }


%left '||' 
%left '&&'
%left '==' '<' '>' '<=' '>='
%left '+' '-' '^+' '++'
%left '*' '/'
%left NEG '!'

%%


-- TODO distinguish between eol and sep (separator)
-- TODO also, this parser will not be able to handle a mix of eol and sep


opt(p):
    p                   { Just $1 }
  |                     { Nothing }


star(p):
  p star(p)       { $1 : $2 }
  |               { [] }

plus(p):
  p star(p)       { $1 : $2 }



Prog:
    star(eol) Expr star(eol) { $2 }
  | star(eol)               { LitString "" }



Expr:
    Ident          { $1 }
  | NonIdentNonSymbolSimpleExpr { $1 }
  | LocalBinding   { $1 }
  | FunDefOrApOrLambda { $1 }
  | MatchExpr      { $1 }
  | DoExpr         { $1 }
  | Module         { $1 }
  | CompoundOrSimpleSymbol  { $1 }
  | InfixOperation   { $1 }
  | IfElse         { $1 }


SimpleExpr:
    Ident               { $1 }
  | NonIdentSimpleExpr  { $1 }


NonIdentSimpleExpr:
    symbol         { LitSymbol $1 [] }
  | NonIdentNonSymbolSimpleExpr { $1 }

NonIdentNonSymbolSimpleExpr:
    int            { LitNumber $1 }
  | string         { LitString $1 }
  | List           { $1 }
  | Record         { $1 }
  | '(' Expr star(TupleNextExpr) ')' {
                  case $3 of
                  [] -> $2
                  es -> LitSymbol tupleSymbolName ($2 : $3) }

TupleNextExpr:
    ',' Expr   { $2 }



FunDefOrApOrLambda:
    NonIdentNonSymbolSimpleExpr plus(SimpleExpr) { FunAp $1 $2 }
  | Ident NonIdentSimpleExpr star(SimpleExpr) { FunAp $1 ($2 : $3)  }
  | Ident Ident FunDefOrCallOrLambdaNext      { let args = $2 : (fst $3) in (snd $3) $1 args }
  | Ident '->' opt(eol) Expr                  { Lambda [varName $1] $4 }


FunDefOrCallOrLambdaNext:
    Ident FunDefOrCallOrLambdaNext             { ($1 : (fst $2), (snd $2)) } -- could still be a fun def or application
  | NonIdentSimpleExpr star(SimpleExpr)        { ($1 : $2, \ a args -> FunAp a args)   }  -- application
  | '=' opt(eol) Expr eol Expr                 { ([], \ a args ->
                                                        LocalBinding (Binding (varName a) (Lambda (map varName args) $3)) $5) } -- fun def
  | '->' opt(eol) Expr                         { ([], \ a args -> Lambda (map varName (a : args)) $3) }
  |                                            { ([], \ a args -> FunAp a args) }



InfixOperation:
    Operand '+' Operand         { FunAp (Var "+") [$1, $3] }
  | Operand '-' Operand         { FunAp (Var "-") [$1, $3] }
  | Operand '/' Operand         { FunAp (Var "/") [$1, $3] }
  | Operand '*' Operand         { FunAp (Var "*") [$1, $3] }
  | Operand '==' Operand        { FunAp (Var "==") [$1, $3] }
  | Operand '<' Operand         { FunAp (Var "<") [$1, $3] }
  | Operand '>' Operand         { FunAp (Var ">") [$1, $3] }
  | Operand '^+' Operand        { FunAp (Var bifStringConcatName) [$1, $3] }
  | Operand '++' Operand        { FunAp (Var bifListConcatName) [$1, $3] }
  | Operand '||' Operand        { FunAp (Var "||") [$1, $3] }
  | Operand '&&' Operand        { FunAp (Var "&&") [$1, $3] }
  | Operand '<=' Operand        { FunAp (Var "<=") [$1, $3] }
  | Operand '>=' Operand        { FunAp (Var ">=") [$1, $3] }
  | '!' Operand                 { FunAp (Var "!") [$2] }
  | '-' Operand %prec NEG       { FunAp (Var "-") [LitNumber 0, $2] }
  -- | Operand operator Operand    { FunAp (Var $2) [$1, $3] }

Operand:
    SimpleExpr     { $1 }
  | InfixOperation { $1 }



List:
    '[' ListNext ']'       { $2 }

ListNext:
    Expr              { LitSymbol listConsSymbolName [$1, LitSymbol listEmptySymbolName []] }
  | Expr ',' ListNext { LitSymbol listConsSymbolName [$1, $3] }
  | Expr '|' Expr     { LitSymbol listConsSymbolName [$1, $3] }
  |                   { LitSymbol listEmptySymbolName [] }


Record:
    '{' RecordBody '}'  { makeRecordSymbol LitSymbol (\ (LitSymbol a _) -> a) $2 }

RecordBody:
    RecordEntry star(RecordNext) { $1 : $2 }

RecordNext:
    ',' RecordEntry { $2 }

RecordEntry:
    id '=' Expr  { (LitSymbol $1 [], $3) }



LocalBinding:
    Binding Expr  { LocalBinding $1 $2 }

Binding:
    id '=' opt(eol) Expr eol   { Binding $1 $4 }


Ident:
    id    { Var $1 }
  | ns Ident  { Qualified $1 $2 } -- TODO use namespace


CompoundOrSimpleSymbol:
    symbol '<' SymbolNext '>' { LitSymbol $1 $3 }
  | symbol { LitSymbol $1 [] }

SymbolNext:
    SimpleExpr                { [$1] }
  | SimpleExpr ',' SymbolNext { $1 : $3 }


Module:
    module opt(eol) star(Definition) end  { Module $3 }
<commit_msg>Adds optional newlines to record parsing<commit_after>{
module Language.Dash.Parser.Parser where

import Language.Dash.Parser.Lexer
import Language.Dash.IR.Ast
import Language.Dash.BuiltIn.BuiltInDefinitions

import Data.List (sortBy)

}

%name       parse
%tokentype  { Token }
%error      { parseError }

%token
  eol       { TEOL }
  '('       { TOpen_Par }
  ')'       { TClose_Par }
  '['       { TOpen_Bracket }
  ']'       { TClose_Bracket }
  '{'       { TOpen_Brace }
  '}'       { TClose_Brace }
  module    { TModule }
  if        { TIf }
  then      { TThen }
  else      { TElse }
  '='       { TDefine }
  symbol    { TSymbol $$ }
  id        { TId $$ }
  ns        { TNamespace $$ }
  -- TODO qid       { TQId $$ }
  string    { TString $$ }
  int       { TInt $$ }
  match     { TMatch }
  do        { TDo }
  '->'      { TArrow_R }
  '<-'      { TArrow_L }
  with      { TWith }
  -- begin     { TBegin }
  end       { TEnd }
  '+'       { TOperator "+" }
  '-'       { TOperator "-" }
  '/'       { TOperator "/" }
  '*'       { TOperator "*" }
  '=='      { TOperator "==" }
  '<'       { TOperator "<" }
  '>'       { TOperator ">" }
  '<='      { TOperator "<=" }
  '>='      { TOperator ">=" }
  '^+'      { TOperator "^+" }
  '++'      { TOperator "++" }
  '||'      { TOperator "||" }
  '&&'      { TOperator "&&" }
  '!'       { TOperator "!" }
  operator  { TOperator $$ }
  '_'       { TUnderscore }
  ','       { TComma }
  '|'       { TVBar }


%left '||' 
%left '&&'
%left '==' '<' '>' '<=' '>='
%left '+' '-' '^+' '++'
%left '*' '/'
%left NEG '!'

%%


-- TODO distinguish between eol and sep (separator)
-- TODO also, this parser will not be able to handle a mix of eol and sep


opt(p):
    p                   { Just $1 }
  |                     { Nothing }


star(p):
  p star(p)       { $1 : $2 }
  |               { [] }

plus(p):
  p star(p)       { $1 : $2 }



Prog:
    star(eol) Expr star(eol) { $2 }
  | star(eol)               { LitString "" }



Expr:
    Ident          { $1 }
  | NonIdentNonSymbolSimpleExpr { $1 }
  | LocalBinding   { $1 }
  | FunDefOrApOrLambda { $1 }
  | MatchExpr      { $1 }
  | DoExpr         { $1 }
  | Module         { $1 }
  | CompoundOrSimpleSymbol  { $1 }
  | InfixOperation   { $1 }
  | IfElse         { $1 }


SimpleExpr:
    Ident               { $1 }
  | NonIdentSimpleExpr  { $1 }


NonIdentSimpleExpr:
    symbol         { LitSymbol $1 [] }
  | NonIdentNonSymbolSimpleExpr { $1 }

NonIdentNonSymbolSimpleExpr:
    int            { LitNumber $1 }
  | string         { LitString $1 }
  | List           { $1 }
  | Record         { $1 }
  | '(' Expr star(TupleNextExpr) ')' {
                  case $3 of
                  [] -> $2
                  es -> LitSymbol tupleSymbolName ($2 : $3) }

TupleNextExpr:
    ',' Expr   { $2 }



FunDefOrApOrLambda:
    NonIdentNonSymbolSimpleExpr plus(SimpleExpr) { FunAp $1 $2 }
  | Ident NonIdentSimpleExpr star(SimpleExpr) { FunAp $1 ($2 : $3)  }
  | Ident Ident FunDefOrCallOrLambdaNext      { let args = $2 : (fst $3) in (snd $3) $1 args }
  | Ident '->' opt(eol) Expr                  { Lambda [varName $1] $4 }


FunDefOrCallOrLambdaNext:
    Ident FunDefOrCallOrLambdaNext             { ($1 : (fst $2), (snd $2)) } -- could still be a fun def or application
  | NonIdentSimpleExpr star(SimpleExpr)        { ($1 : $2, \ a args -> FunAp a args)   }  -- application
  | '=' opt(eol) Expr eol Expr                 { ([], \ a args ->
                                                        LocalBinding (Binding (varName a) (Lambda (map varName args) $3)) $5) } -- fun def
  | '->' opt(eol) Expr                         { ([], \ a args -> Lambda (map varName (a : args)) $3) }
  |                                            { ([], \ a args -> FunAp a args) }



InfixOperation:
    Operand '+' Operand         { FunAp (Var "+") [$1, $3] }
  | Operand '-' Operand         { FunAp (Var "-") [$1, $3] }
  | Operand '/' Operand         { FunAp (Var "/") [$1, $3] }
  | Operand '*' Operand         { FunAp (Var "*") [$1, $3] }
  | Operand '==' Operand        { FunAp (Var "==") [$1, $3] }
  | Operand '<' Operand         { FunAp (Var "<") [$1, $3] }
  | Operand '>' Operand         { FunAp (Var ">") [$1, $3] }
  | Operand '^+' Operand        { FunAp (Var bifStringConcatName) [$1, $3] }
  | Operand '++' Operand        { FunAp (Var bifListConcatName) [$1, $3] }
  | Operand '||' Operand        { FunAp (Var "||") [$1, $3] }
  | Operand '&&' Operand        { FunAp (Var "&&") [$1, $3] }
  | Operand '<=' Operand        { FunAp (Var "<=") [$1, $3] }
  | Operand '>=' Operand        { FunAp (Var ">=") [$1, $3] }
  | '!' Operand                 { FunAp (Var "!") [$2] }
  | '-' Operand %prec NEG       { FunAp (Var "-") [LitNumber 0, $2] }
  -- | Operand operator Operand    { FunAp (Var $2) [$1, $3] }

Operand:
    SimpleExpr     { $1 }
  | InfixOperation { $1 }



List:
    '[' ListNext ']'       { $2 }

ListNext:
    Expr              { LitSymbol listConsSymbolName [$1, LitSymbol listEmptySymbolName []] }
  | Expr ',' ListNext { LitSymbol listConsSymbolName [$1, $3] }
  | Expr '|' Expr     { LitSymbol listConsSymbolName [$1, $3] }
  |                   { LitSymbol listEmptySymbolName [] }


Record:
    '{' opt(eol) RecordBody '}'  { makeRecordSymbol LitSymbol (\ (LitSymbol a _) -> a) $3 }

RecordBody:
    RecordEntry star(RecordNext) { $1 : $2 }

RecordNext:
    ',' opt(eol) RecordEntry { $3 }

RecordEntry:
    id opt(eol) '=' Expr opt(eol)  { (LitSymbol $1 [], $4) }



LocalBinding:
    Binding Expr  { LocalBinding $1 $2 }

Binding:
    id '=' opt(eol) Expr eol   { Binding $1 $4 }


Ident:
    id    { Var $1 }
  | ns Ident  { Qualified $1 $2 } -- TODO use namespace


CompoundOrSimpleSymbol:
    symbol '<' SymbolNext '>' { LitSymbol $1 $3 }
  | symbol { LitSymbol $1 [] }

SymbolNext:
    SimpleExpr                { [$1] }
  | SimpleExpr ',' SymbolNext { $1 : $3 }


Module:
    module opt(eol) star(Definition) end  { Module $3 }

{'fraction_non_alphanumeric': 0.1767034447907016, 'fraction_numerical': 0.01819253769055841, 'mean_word_length': 1.7346844771994472, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3290562', 'n_tokens_mistral': 4413, 'n_tokens_neox': 4285, 'n_words': 975}