<commit_before>\ File word set

: BIN 4 or ;
: R/O 1 ;
: W/O 2 ;
: R/W 3 ;

\ Possible cases:
\ - If we parsed some content:
\   - Check the last value parsed. If it was ), done.
\   - If not, we're at EOL so refill and loop.
\ - Parse length of 0:
\   - If the source length is 0, blank line; refill and loop.
\   - Source length non-zero, check previous. if ), done. If not, refill.
: (
  \ Fall back to the original if this isn't a file.
  source-id 0 <= IF 41 parse 2drop EXIT THEN
  BEGIN
    41 parse nip ( len )
    IF \ Parsed something. Check if last character was ')'
      >IN @ 1- source drop + c@   41 = IF EXIT THEN
    ELSE \ Empty parse. Check if this line is empty.
      source 0=   >IN @ 0=   or IF ( c-addr )
        drop refill 0= IF EXIT THEN \ Refill, bail if EOF.
      ELSE
        \ Nonempty line, so check last character parsed.
        >IN @ 1- + c@   41 = IF EXIT THEN
        refill 0= IF EXIT THEN
      THEN
    THEN
  AGAIN
; IMMEDIATE


: INCLUDED ( i*x c-addr u -- j*x )
  2dup r/o open-file ABORT" Could not open file" >R
  \ Record this file in the inclusion names list.
  (included-file-list) @   here (included-file-list) !
  ( c-addr u list-head    R: fd )
  ,   dup ,   here swap move align ( R: fd )
  R> include-file
;

: INCLUDE ( i*x "name" -- j*x ) parse-name included ;

\ NB: REQUIRE(D) files should have no net stack impact; INCLUDE(D) ones can.<commit_msg>Fix allocation error in included file list<commit_after>\ File word set

: BIN 4 or ;
: R/O 1 ;
: W/O 2 ;
: R/W 3 ;

\ Possible cases:
\ - If we parsed some content:
\   - Check the last value parsed. If it was ), done.
\   - If not, we're at EOL so refill and loop.
\ - Parse length of 0:
\   - If the source length is 0, blank line; refill and loop.
\   - Source length non-zero, check previous. if ), done. If not, refill.
: (
  \ Fall back to the original if this isn't a file.
  source-id 0 <= IF 41 parse 2drop EXIT THEN
  BEGIN
    41 parse nip ( len )
    IF \ Parsed something. Check if last character was ')'
      >IN @ 1- source drop + c@   41 = IF EXIT THEN
    ELSE \ Empty parse. Check if this line is empty.
      source 0=   >IN @ 0=   or IF ( c-addr )
        drop refill 0= IF EXIT THEN \ Refill, bail if EOF.
      ELSE
        \ Nonempty line, so check last character parsed.
        >IN @ 1- + c@   41 = IF EXIT THEN
        refill 0= IF EXIT THEN
      THEN
    THEN
  AGAIN
; IMMEDIATE


: INCLUDED ( i*x c-addr u -- j*x )
  2dup r/o open-file ABORT" Could not open file" >R
  \ Record this file in the inclusion names list.
  (included-file-list) @   here (included-file-list) !
  ( c-addr u list-head    R: fd )
  ,   dup ,   here swap    dup allot    move align ( R: fd )
  R> include-file
;

: INCLUDE ( i*x "name" -- j*x ) parse-name included ;

\ NB: REQUIRE(D) files should have no net stack impact; INCLUDE(D) ones can.
{'fraction_non_alphanumeric': 0.1064053202660133, 'fraction_numerical': 0.016100805040252013, 'mean_word_length': 2.0897297297297297, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5405522', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1000, 'n_words': 427}