<commit_before>require 'zippo/central_directory_unpacker'
require 'zippo/central_directory_entries_unpacker'

module Zippo
  class CentralDirectoryParser
    SIGNATURE = 0x06054b50
    PACKED_SIGNATURE = [SIGNATURE].pack('L')
    MAX_COMMENT_LENGTH = 1<<16
    # reads size from the specified offset
    # if offset is negative, will offset from EOF
    def read size, offset
      @io.seek offset, (offset < 0 ? IO::SEEK_END : IO::SEEK_SET)
      @io.read size
    end
    # reads from the specified offset until EOF
    def read_from offset
      read nil, offset
    end
    def initialize(io)
      @io = io
    end
    def end_of_cd_record
      EndCdRecord::Unpacker.new(read_from end_of_cd_record_position).unpack
    end
    def end_of_cd_record_position
      # XXX implement optimised scanning at -22 position
      scan_size = [@io.size, MAX_COMMENT_LENGTH].min
      (@io.size - scan_size) + read_from(-scan_size).rindex(PACKED_SIGNATURE).tap do |pos|
        raise "End of Central Directory Record not found" unless pos
      end
    end
    def cd_file_headers
      CentralDirectoryEntriesUnpacker.new(read end_of_cd_record.cd_size, end_of_cd_record.cd_offset).unpack
    end
  end
end<commit_msg>Refactor and optimize<commit_after>require 'zippo/central_directory_unpacker'
require 'zippo/central_directory_entries_unpacker'

module Zippo
  class CentralDirectoryParser
    MAX_COMMENT_LENGTH = 1<<16

    def initialize(io)
      @io = io
    end

    def end_of_cd_record
      @end_of_cd_record ||= EndCdRecord::Unpacker.new(read_from end_of_cd_record_position).unpack
    end

    def cd_file_headers
      @cd_file_headers ||= CentralDirectoryEntriesUnpacker.new(read end_of_cd_record.cd_size, end_of_cd_record.cd_offset).unpack
    end

    def end_of_cd_record_position
      # XXX implement optimised scanning at -22 position
      [44, 22].each do |pos|
        return @io.size - pos if (read 4, -pos) == EndCdRecord::PACKED_SIGNATURE
      end

      scan_size = [@io.size, MAX_COMMENT_LENGTH].min
      (@io.size - scan_size) + read_from(-scan_size).rindex(EndCdRecord::PACKED_SIGNATURE).tap do |pos|
        raise "End of Central Directory Record not found" unless pos
      end
    end

    private
    # reads size from the specified offset
    # if offset is negative, will offset from EOF
    def read size, offset
      @io.seek offset, (offset < 0 ? IO::SEEK_END : IO::SEEK_SET)
      @io.read size
    end

    # reads from the specified offset until EOF
    def read_from offset
      read nil, offset
    end
  end
end
{'fraction_non_alphanumeric': 0.07279843444227006, 'fraction_numerical': 0.009784735812133072, 'mean_word_length': 3.252911813643927, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3731174', 'n_tokens_mistral': 863, 'n_tokens_neox': 829, 'n_words': 217}