<commit_before>require 'rugzip/header'
require 'zlib'

module Rugzip
  class Decompressor
    TRAILER_LEN = 8
    
    def initialize(input, output)
      @in, @out = input, output
    end

    def decompress
      reset
      parse_header
      parse_fextra
      parse_fname
      parse_fcomment
      parse_fhcrc
      inflate_data
      parse_trailer
    end
    
    private
    
    def inflate_data
      # find the # of bytes needed to read the compressed blocks and read them
      data_len = @in.size - TRAILER_LEN - @in.pos
      deflated = @in.read(data_len)
      
      # inflate the compressed blocks
      zstream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
      inflated = zstream.inflate(deflated)
      zstream.finish
      zstream.close
      
      @out.write(inflated)
    end
    
    def parse_fcomment
      return unless @header.flg.fcomment?
      # TODO
    end
    
    def parse_fextra
      return unless @header.flg.fextra?
      
      xlen = @in.read(2).unpack('S').first
      xtra = @in.read(xlen)
      # TODO: parse xtra
    end
    
    def parse_fhcrc
      return unless @header.flg.fhcrc?
    end
    
    def parse_fname
      return unless @header.flg.fname?
      # TODO
    end
    
    def parse_header
      @header = Header.new(@in.read(Header::STATIC_LEN))
      raise 'yo, make an error for this later' unless @header.valid?
    end
    
    def parse_trailer
      # TODO<commit_msg>Modify #inflate_data to use a buffer with a limited size.<commit_after>require 'rugzip/header'
require 'zlib'

module Rugzip
  class Decompressor
    BUF_LEN     = 4096
    TRAILER_LEN = 8
    
    def initialize(input, output)
      @in, @out = input, output
    end

    def decompress
      reset
      parse_header
      parse_fextra
      parse_fname
      parse_fcomment
      parse_fhcrc
      inflate_data
      parse_trailer
    end
    
    private
    
    def inflate_data
      zstream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
      
      # find the # of bytes needed to read the compressed blocks
      data_len = @in.size - TRAILER_LEN - @in.pos
      
      # read the deflated blocks and inflate them
      bytes_read = 0
      while bytes_read < data_len
        bytes_left = data_len - bytes_read
        buflen = [bytes_left, BUF_LEN].min
        
        deflated = @in.read(buflen)
        inflated = zstream.inflate(deflated)
        
        @out.write(inflated)
        bytes_read += deflated.bytesize
      end
      
      zstream.finish
      zstream.close
    end
    
    def parse_fcomment
      return unless @header.flg.fcomment?
      # TODO
    end
    
    def parse_fextra
      return unless @header.flg.fextra?
      
      xlen = @in.read(2).unpack('S').first
      xtra = @in.read(xlen)
      # TODO: parse xtra
    end
    
    def parse_fhcrc
      return unless @header.flg.fhcrc?
    end
    
    def parse_fname
      return unless @header.flg.fname?
      # TODO
    end
    
    def parse_header
      @header = Header.new(@in.read(Header::STATIC_LEN))
      raise 'yo, make an error for this later' unless @header.valid?
    end
    
    def parse_trailer
      # TODO
{'fraction_non_alphanumeric': 0.07154213036565978, 'fraction_numerical': 0.002861685214626391, 'mean_word_length': 1.8968692449355433, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2853667', 'n_tokens_mistral': 1062, 'n_tokens_neox': 1043, 'n_words': 252}