<commit_before>! Copyright (C) 2007 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel arrays strings math.parser sequences sequences.deep
peg peg.ebnf peg.parsers memoize namespaces math ;
IN: peg.javascript

#! Grammar for JavaScript. Based on OMeta-JS example from:
#! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler 

USE: prettyprint

TUPLE: ast-keyword value ;
TUPLE: ast-name value ;
TUPLE: ast-number value ;
TUPLE: ast-string value ;
TUPLE: ast-cond-expr condition then else ;
TUPLE: ast-set lhs rhs ;
TUPLE: ast-get value ;
TUPLE: ast-mset lhs rhs operator ;
TUPLE: ast-binop lhs rhs operator ;
TUPLE: ast-unop expr operator ;
TUPLE: ast-postop expr operator ;
TUPLE: ast-preop expr operator ;
TUPLE: ast-getp index expr ;
TUPLE: ast-send method expr args ;
TUPLE: ast-call expr args ;
TUPLE: ast-this ;
TUPLE: ast-new name args ;
TUPLE: ast-array values ;
TUPLE: ast-json bindings ;
TUPLE: ast-binding name value ;
TUPLE: ast-func fs body ;
TUPLE: ast-var name value ;
TUPLE: ast-begin statements ;
TUPLE: ast-if condition true false ;
TUPLE: ast-while condition statements ;
TUPLE: ast-do-while statements condition ;
TUPLE: ast-for i c u statements ;
TUPLE: ast-for-in v e statements ;
TUPLE: ast-switch expr statements ;
TUPLE: ast-break ;
TUPLE: ast-continue ;
TUPLE: ast-throw e ;
TUPLE: ast-try t e c f ;
TUPLE: ast-return e ;
TUPLE: ast-case c cs ;
TUPLE: ast-default cs ;

EBNF: javascript
Letter            = [a-zA-Z]
Digit             = [0-9]
Digits            = (Digit)+
SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]]
MultiLineComment  = "/*" (!("*/") .)* "*/" => [[ ignore ]]
Space             = " " | "\t" | "\n" | SingleLineComment | MultiLineComment
Spaces            = (Space)* => [[ ignore ]]
NameFirst         = Letter | "$" | "_"
NameRest          = NameFirst | Digit
iName             = NameFirst (NameRest)* => [[ first2 swap prefix >string ]]
Keyword           =  ("break"
                    | "case"
                    | "catch"
                    | "continue"
                    | "default"
                    | "delete"
                    | "do"
                    | "else"
                    | "finally"
                    | "for"
                    | "function"
                    | "if"
                    | "in"
                    | "instanceof"
                    | "new"<commit_msg>Split out javascript tokenizer<commit_after>! Copyright (C) 2007 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel arrays strings math.parser sequences sequences.deep
peg peg.ebnf peg.parsers memoize namespaces math ;
IN: peg.javascript

#! Grammar for JavaScript. Based on OMeta-JS example from:
#! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler 

USE: prettyprint

TUPLE: ast-keyword value ;
TUPLE: ast-name value ;
TUPLE: ast-number value ;
TUPLE: ast-string value ;
TUPLE: ast-cond-expr condition then else ;
TUPLE: ast-set lhs rhs ;
TUPLE: ast-get value ;
TUPLE: ast-mset lhs rhs operator ;
TUPLE: ast-binop lhs rhs operator ;
TUPLE: ast-unop expr operator ;
TUPLE: ast-postop expr operator ;
TUPLE: ast-preop expr operator ;
TUPLE: ast-getp index expr ;
TUPLE: ast-send method expr args ;
TUPLE: ast-call expr args ;
TUPLE: ast-this ;
TUPLE: ast-new name args ;
TUPLE: ast-array values ;
TUPLE: ast-json bindings ;
TUPLE: ast-binding name value ;
TUPLE: ast-func fs body ;
TUPLE: ast-var name value ;
TUPLE: ast-begin statements ;
TUPLE: ast-if condition true false ;
TUPLE: ast-while condition statements ;
TUPLE: ast-do-while statements condition ;
TUPLE: ast-for i c u statements ;
TUPLE: ast-for-in v e statements ;
TUPLE: ast-switch expr statements ;
TUPLE: ast-break ;
TUPLE: ast-continue ;
TUPLE: ast-throw e ;
TUPLE: ast-try t e c f ;
TUPLE: ast-return e ;
TUPLE: ast-case c cs ;
TUPLE: ast-default cs ;

EBNF: tokenizer 
Letter            = [a-zA-Z]
Digit             = [0-9]
Digits            = (Digit)+
SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]]
MultiLineComment  = "/*" (!("*/") .)* "*/" => [[ ignore ]]
Space             = " " | "\t" | "\n" | SingleLineComment | MultiLineComment
Spaces            = (Space)* => [[ ignore ]]
NameFirst         = Letter | "$" | "_"
NameRest          = NameFirst | Digit
iName             = NameFirst (NameRest)* => [[ first2 swap prefix >string ]]
Keyword           =  ("break"
                    | "case"
                    | "catch"
                    | "continue"
                    | "default"
                    | "delete"
                    | "do"
                    | "else"
                    | "finally"
                    | "for"
                    | "function"
                    | "if"
                    | "in"
                    | "instanceof"
                    | "new"
                    | "return"
                    | "switch"
                    | "this"
                    | "throw"
                    | "try"
                    | "typeof"
                    | "var"
                    | "void"
                    | "while"
                    | "with") => [[ ast-keyword boa ]]
Name              = !(Keyword) (iName):n => [[ n ast-name boa ]]
Number            =   Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]]
                    | Digits => [[ >string string>number ast-number boa ]]  

EscapeChar        =   "\\n" => [[ 10 ]] 
                    | "\\r" => [[ 13 ]]
                    | "\\t" => [[ 9 ]]
StringChars1       = (EscapeChar | !('"""') .)* => [[ >string ]]
StringChars2       = (EscapeChar | !('"') .)* => [[ >string ]]
StringChars3       = (EscapeChar | !("'") .)* => [[ >string ]]
Str                =   '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]]
                     | '"' StringChars2:cs '"' => [[ cs ast-string boa ]]
                     | "'" StringChars3:cs "'" => [[ cs ast-string boa ]]
Special            =   "("   | ")"   | "{"   | "}"   | "["   | "]"   | ","   | ";"
                     | "?"   | ":"   | "!==" | "~="  | "===" | "=="  | "="   | ">="
                     | ">"   | "<="  | "<"   | "++"  | "+="  | "+"   | "--"  | "-="
                     | "-"   | "*="  | "*"   | "/="  | "/"   | "%="  | "%"   | "&&="
                     | "&&"  | "||=" | "||"  | "."   | "!"
Tok                = Spaces (Name | Keyword | Number | Str | Special )
Toks               = (Tok)* Spaces 
;EBNF

EBNF: javascript
Letter            = [a-zA-Z]
Digit             = [0-9]
Digits            = (Digit)+
SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]]
MultiLineComment  = "/*" (!("*/") .)* "*/" => [[ ignore ]]
Space             = " " | "\t" | "\n" | SingleLineComment | MultiLineComment
Spaces            = (Space)* => [[ ignore ]]
NameFirst         = Letter | "$" | "_"
NameRest          = NameFirst | Digit
iName             = NameFirst (NameRest)* => [[ first2 swap prefix >string ]]
Keyword           =  ("break"
                    | "case"
                    | "catch"
                    | "continue"
                    | "default"
                    | "delete"
                    | "do"
                    | "else"
                    | "finally"
                    | "for"
                    | "function"
                    | "if"
                    | "in"
                    | "instanceof"
                    | "new"
{'fraction_non_alphanumeric': 0.15913102511880517, 'fraction_numerical': 0.003937542430414121, 'mean_word_length': 1.5764253235396992, 'pattern_counts': {'":': 1, '<': 7, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3417495', 'n_tokens_mistral': 2443, 'n_tokens_neox': 2330, 'n_words': 573}