<commit_before>! Copyright (C) 2007, 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel sequences strings namespaces math assocs shuffle 
       vectors arrays combinators.lib math.parser match
       unicode.categories sequences.lib compiler.units parser
       words quotations effects memoize accessors locals ;
IN: peg

USE: prettyprint

TUPLE: parse-result remaining ast ;

TUPLE: parser id compiled ;
M: parser equal? [ id>> ] 2apply = ;
C: <parser> parser

SYMBOL: ignore 

: <parse-result> ( remaining ast -- parse-result )
  parse-result construct-boa ;

SYMBOL: packrat
SYMBOL: pos
SYMBOL: input
SYMBOL: fail
SYMBOL: lrstack
SYMBOL: heads

TUPLE: memo-entry ans pos ;
C: <memo-entry> memo-entry

TUPLE: left-recursion seed rule head next ;
C: <left-recursion> left-recursion
 
TUPLE: peg-head rule involved-set eval-set ;
C: <head> peg-head

: rule-parser ( rule -- parser ) 
  #! A rule is the parser compiled down to a word. It has
  #! a "peg" property containing the original parser.
  "peg" word-prop ;

: input-slice ( -- slice )
  #! Return a slice of the input from the current parse position<commit_msg>Fix hashcode* on parsers<commit_after>! Copyright (C) 2007, 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel sequences strings namespaces math assocs shuffle 
       vectors arrays combinators.lib math.parser match
       unicode.categories sequences.lib compiler.units parser
       words quotations effects memoize accessors locals ;
IN: peg

USE: prettyprint

TUPLE: parse-result remaining ast ;

TUPLE: parser id compiled ;
M: parser equal? [ id>> ] 2apply = ;
M: parser hashcode* ( depth obj -- code )
  id>> hashcode* ;
C: <parser> parser

SYMBOL: ignore 

: <parse-result> ( remaining ast -- parse-result )
  parse-result construct-boa ;

SYMBOL: packrat
SYMBOL: pos
SYMBOL: input
SYMBOL: fail
SYMBOL: lrstack
SYMBOL: heads

TUPLE: memo-entry ans pos ;
C: <memo-entry> memo-entry

TUPLE: left-recursion seed rule head next ;
C: <left-recursion> left-recursion
 
TUPLE: peg-head rule involved-set eval-set ;
C: <head> peg-head

: rule-parser ( rule -- parser ) 
  #! A rule is the parser compiled down to a word. It has
  #! a "peg" property containing the original parser.
  "peg" word-prop ;

: input-slice ( -- slice )
  #! Return a slice of the input from the current parse position
{'fraction_non_alphanumeric': 0.09949622166246852, 'fraction_numerical': 0.007556675062972292, 'mean_word_length': 3.654296875, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4782679', 'n_tokens_mistral': 765, 'n_tokens_neox': 729, 'n_words': 280}