<commit_before>module LZW

  class Simple
    def compress ( data )
      LZW::Compressor.new.compress( data )
    end

    def decompress ( data )
      LZW::Decompressor.new.decompress( data )
    rescue
      nil
    end
  end
  
  class Compressor
    attr_reader :block_mode, :big_endian, :max_code_size, :init_code_size

    def initialize (
      block_mode:     true,
      big_endian:     nil,
      init_code_size: 9,
      max_code_size:  16
    )
      @block_mode = block_mode
      
      if big_endian.nil?
        @big_endian = LZW::big_endian
      end

      if init_code_size > max_code_size 
        raise "init_code_size must be less than or equal to max_code_size"
      end

      if max_code_size > 16
        raise "max_code_size must be 16 or less"
      end

      if init_code_size < 9
        raise "init_code_size must be greater than 8"
      end

      @init_code_size = init_code_size
      @max_code_size  = max_code_size
    end

    def compress( data )
      self.reset()

    end

    def reset
      @code_table = {}
      @code_size  = @init_code_size
      @next_code  = 257
    end

  end

  class Decompressor
    def decompress ( data )
      
    end
  end


  def self.big_endian
    [1].pack("I") == [1].pack("N")
  end

end

<commit_msg>add constants and magic/header setup. dump raw data after header to keep naive test failing<commit_after>module LZW

  MAGIC      = "\037\235".force_encoding('BINARY')
  MASK_BITS  = 0x1f
  MASK_BLOCK = 0x80
  RESET_CODE = 256

  class Simple
    def compress ( data )
      LZW::Compressor.new.compress( data )
    end

    def decompress ( data )
      LZW::Decompressor.new.decompress( data )
    rescue
      nil
    end
  end
  
  class Compressor
    attr_reader :block_mode, :big_endian, :max_code_size, :init_code_size

    def initialize (
      block_mode:     true,
      big_endian:     nil,
      init_code_size: 9,
      max_code_size:  16
    )
      @block_mode     = block_mode
      @init_code_size = init_code_size
      @max_code_size  = max_code_size
      
      if big_endian.nil?
        @big_endian = LZW::big_endian
      end

      if init_code_size > max_code_size 
        raise "init_code_size must be less than or equal to max_code_size"
      end

      if max_code_size > 16
        raise "max_code_size must be 16 or less"
      end

      if init_code_size < 9
        raise "init_code_size must be greater than 8"
      end

      @buf     = magic
      @buf_pos = @buf.size * 8
    end

    def compress( data )
      reset

      @buf << data
    end

    private

    def reset
      @code_table = {}
      @code_size  = @init_code_size
      @next_code  = 257
    end

    def magic
      MAGIC + (
        ( @max_code_size & MASK_BITS ) |
        ( @block_mode ? MASK_BLOCK : 0 )
      ).chr
    end

  end

  class Decompressor
    def decompress ( data )
      
    end
  end


  def self.big_endian
    [1].pack("I") == [1].pack("N")
  end

end


{'fraction_non_alphanumeric': 0.0719327731092437, 'fraction_numerical': 0.014789915966386555, 'mean_word_length': 1.9349112426035502, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2670848', 'n_tokens_mistral': 1099, 'n_tokens_neox': 1022, 'n_words': 262}