<commit_before>! Copyright (C) 2006 <NAME>
! See http://factorcode.org/license.txt for BSD license.
USING: kernel math sequences sequences.private namespaces
words io io.binary io.files io.streams.string quotations ;
IN: io.crc32

: crc32-polynomial HEX: edb88320 ; inline

! Generate the table at load time and define a new word with it,
! instead of using a variable, so that the compiler can inline
! the call to nth-unsafe
DEFER: crc32-table inline

\ crc32-table
256 [
    8 [
        dup even? >r 2/ r> [ crc32-polynomial bitxor ] unless
    ] times >bignum
] map
1quotation define-inline

: (crc32) ( crc ch -- crc )
    >bignum dupd bitxor
    mask-byte crc32-table nth-unsafe >bignum
    swap -8 shift bitxor ; inline

: crc32 ( seq -- n )
    >r HEX: ffffffff dup r> [ (crc32) ] each bitxor ;

: file-crc32 ( path -- n ) file-contents crc32 ;

: lines-crc32 ( seq -- n )
    HEX: ffffffff tuck [ [ (crc32) ] each ] reduce bitxor ;
<commit_msg>lines-crc32 now crcs a \n at the end of each line.  it is still off by one \n if the last line doesn't have a newline, but it's good enough for the parser<commit_after>! Copyright (C) 2006 <NAME>
! See http://factorcode.org/license.txt for BSD license.
USING: kernel math sequences sequences.private namespaces
words io io.binary io.files io.streams.string quotations ;
IN: io.crc32

: crc32-polynomial HEX: edb88320 ; inline

! Generate the table at load time and define a new word with it,
! instead of using a variable, so that the compiler can inline
! the call to nth-unsafe
DEFER: crc32-table inline

\ crc32-table
256 [
    8 [
        dup even? >r 2/ r> [ crc32-polynomial bitxor ] unless
    ] times >bignum
] map
1quotation define-inline

: (crc32) ( crc ch -- crc )
    >bignum dupd bitxor
    mask-byte crc32-table nth-unsafe >bignum
    swap -8 shift bitxor ; inline

: crc32 ( seq -- n )
    >r HEX: ffffffff dup r> [ (crc32) ] each bitxor ;

: file-crc32 ( path -- n ) file-contents crc32 ;

: lines-crc32 ( seq -- n )
    HEX: ffffffff tuck [
        [ (crc32) ] each CHAR: \n (crc32)
    ] reduce bitxor ;

{'fraction_non_alphanumeric': 0.10260115606936417, 'fraction_numerical': 0.04238921001926782, 'mean_word_length': 2.963740458015267, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5800535', 'n_tokens_mistral': 812, 'n_tokens_neox': 744, 'n_words': 269}