<commit_before>! Copyright (C) 2009 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors combinators io kernel math namespaces
prettyprint sequences vectors ;
QUALIFIED-WITH: bitstreams bs
IN: compression.lzw

TUPLE: lzw
input
output
table
code
old-code
initial-code-size
code-size
clear-code
end-of-information-code ;

TUPLE: tiff-lzw < lzw ;
TUPLE: gif-lzw < lzw ;

: initial-uncompress-table ( size -- seq )
    iota [ 1vector ] V{ } map-as ;

: reset-lzw-uncompress ( lzw -- lzw )
    dup end-of-information-code>> 1 + initial-uncompress-table >>table
    dup initial-code-size>> >>code-size ;

: <lzw-uncompress> ( input code-size class -- obj )
    new
        swap >>code-size
        dup code-size>> >>initial-code-size
        dup code-size>> 1 - 2^ >>clear-code
        dup clear-code>> 1 + >>end-of-information-code
        swap >>input
        BV{ } clone >>output
        reset-lzw-uncompress ;

ERROR: not-in-table value ;<commit_msg>Make sure we're not reading 0 bits in the lzw decompression loop -- this causes a hang<commit_after>! Copyright (C) 2009 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors combinators io kernel math namespaces
prettyprint sequences vectors ;
QUALIFIED-WITH: bitstreams bs
IN: compression.lzw

TUPLE: lzw
input
output
table
code
old-code
initial-code-size
code-size
clear-code
end-of-information-code ;

TUPLE: tiff-lzw < lzw ;
TUPLE: gif-lzw < lzw ;

: initial-uncompress-table ( size -- seq )
    iota [ 1vector ] V{ } map-as ;

: reset-lzw-uncompress ( lzw -- lzw )
    dup end-of-information-code>> 1 + initial-uncompress-table >>table
    dup initial-code-size>> >>code-size ;

ERROR: code-size-zero ;

: <lzw-uncompress> ( input code-size class -- obj )
    new
        swap [ code-size-zero ] when-zero >>code-size
        dup code-size>> >>initial-code-size
        dup code-size>> 1 - 2^ >>clear-code
        dup clear-code>> 1 + >>end-of-information-code
        swap >>input
        BV{ } clone >>output
        reset-lzw-uncompress ;

ERROR: not-in-table value ;
{'fraction_non_alphanumeric': 0.12878423834694858, 'fraction_numerical': 0.00913022585295531, 'mean_word_length': 2.898876404494382, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3200145', 'n_tokens_mistral': 734, 'n_tokens_neox': 703, 'n_words': 197}