<commit_before>class Dockly::TarDiff
  include Dockly::Util::Logger::Mixin

  # Tar header format for a ustar tar
  HEADER_UNPACK_FORMAT  = "Z100A8A8A8A12A12A8aZ100A6A2Z32Z32A8A8Z155"

  logger_prefix '[dockly tar_diff]'

  attr_reader :base, :target, :output, :target_enum, :base_enum

  def initialize(base, target, output)
    @base, @target, @output = base, target, output

    @base_enum = to_enum(:read_header, this.base)
    @target_enum = to_enum(:read_header, this.target)
  end

  def skip(target, size)
    if target.respond_to?(:seek)
      target.seek(size, IO::SEEK_CUR)
    else
      target.read(size)
    end
  end

  def write_tar_section(output, header, size, remainder, target)
    output.write(header)
    quick_write(output, size, target)
    output.write("\0" * remainder)
  end

  def quick_write(output, size, target)
    while size > 0
      bread = read.read([size, 4096].min)
      output.write(bread)
      raise UnexpectedEOF if read.eof?
      size -= bread.size
    end
  end

  def read_header(target)
    loop do
      return if target.eof?

      # Tar header is 512 bytes large
      data = target.read(512)
      fields = data.unpack(HEADER_UNPACK_FORMAT)
      name = fields[0]
      size = fields[4].oct
      mtime = fields[5].oct
      prefix = fields[15]

      empty = (data == "\0" * 512)
      remainder = (512 - (size % 512)) % 512

      yield data, name, prefix, mtime, size, remainder, empty

      skip(target, remainder)
    end
  end

  def diff_once
    begin
      target_header, target_name,  \
      target_prefix, target_mtime,  \
      target_size, target_remainder, \
      target_empty                    = target_enum.peek
    rescue StopIteration
      puts "Done with new file"
      return false
    end

    return false if target_empty

    begin
      _, base_name, base_prefix, base_mtime,\
      base_size, _, base_empty  = base_enum.peek
    rescue StopIteration
      puts "Done with base file"
      write_tar_section(output, target_header, target_size, target_remainder, target)
      target_enum.next
      return true
    end

    if base_empty
      write_tar_section(output, target_header, target_size, target_remainder, target)
      target_enum.next
      return true
    end

    target_full_name = File.join(target_prefix, target_name)
    base_full_name = File.join(base_prefix, base_name)

    if (target_full_name < base_full_name)
      write_tar_section(output, target_header, target_size, target_remainder, target)
      target_enum.next
    elsif (base_full_name < target_full_name)
      skip(base, base_size)
      base_enum.next
    elsif (target_mtime != base_mtime) || (target_size != base_size)
      write_tar_section(output, target_header, target_size, target_remainder, target)
      target_enum.next
    else
      skip(target, target_size)
      target_enum.next
      skip(base, base_size)
      base_enum.next
    end

    return true
  end

  def diff
    loop do
      break unless diff_once
    end
  end
end<commit_msg>TarDiff now accepts a chunk and acts upon the size of the chunk<commit_after>class Dockly::TarDiff
  include Dockly::Util::Logger::Mixin

  # Tar header format for a ustar tar
  HEADER_UNPACK_FORMAT  = "Z100A8A8A8A12A12A8aZ100A6A2Z32Z32A8A8Z155"

  logger_prefix '[dockly tar_diff]'

  attr_reader :base, :output, :base_enum, :base_data, :target_data

  attr_accessor :base_data, :target_data, :previous_chunk

  def initialize(base, output)
    @previous_chunk = ""
    @base, @output, @input = base, output, StringIO.new

    @base_data = @target_data = nil
  end

  def skip(io, size)
    if io.respond_to?(:seek)
      io.seek(size, IO::SEEK_CUR)
    else
      io.read(size)
    end
  end

  def write_tar_section(output, header, size, remainder, input)
    self.target_data = nil
    output.write(header)
    quick_write(output, size, input)
    skip(input, target_remainder)
    output.write("\0" * remainder)
    self.previous_chunk = input.read
  end

  def quick_write(output, size, target)
    while size > 0
      bread = target.read([size, 4096].min)
      output.write(bread)
      raise UnexpectedEOF if read.eof?
      size -= bread.size
    end
  end

  def read_header(input)
    return if input.eof?

    # Tar header is 512 bytes large
    data = input.read(512)
    fields = data.unpack(HEADER_UNPACK_FORMAT)
    name = fields[0]
    size = fields[4].oct
    mtime = fields[5].oct
    prefix = fields[15]

    empty = (data == "\0" * 512)
    remainder = (512 - (size % 512)) % 512

    return data, name, prefix, mtime, size, remainder, empty
  end

  # Convert from enum style with yield to return style:
  #   - Must be able to allow for less than the size of a full header and full
  #     file from the input
  #   - Operate using StringIO
  def process(raw_input)
    input = StringIO.new(previous_chunk + raw_input)

    unless target_data || input.size > 512
      self.previous_chunk = input.read
      return false
    end

    self.target_data ||= read_header(input)

    target_header, target_name,  \
    target_prefix, target_mtime,  \
    target_size, target_remainder, \
    target_empty                    = target_data

    if target_empty || (input.length - input.pos) > (target_size + target_remainder)
      self.previous_chunk = input.read
      return false
    end

    if base_data || (base.length - base.pos) > 512
      self.base_data ||= read_header(base)

      _, base_name, base_prefix, base_mtime, base_size, _, base_empty = base_data
    else
      write_tar_section(output, target_header, target_size, target_remainder, input)
      return true
    end

    if base_empty
      write_tar_section(output, target_header, target_size, target_remainder, input)
      return true
    end

    target_full_name = File.join(target_prefix, target_name)
    base_full_name = File.join(base_prefix, base_name)

    if (target_full_name < base_full_name)
      write_tar_section(output, target_header, target_size, target_remainder, input)
    elsif (base_full_name < target_full_name)
      skip(base, base_size + base_remainder)
      self.base_data = nil
      self.previous_chunk = input.read
    elsif (target_mtime != base_mtime) || (target_size != base_size)
      write_tar_section(output, target_header, target_size, target_remainder, input)
    else
      skip(input, target_size + target_remainder)
      self.previous_chunk = input.read
      self.target_data = nil
      skip(base, base_size + base_remainder)
      self.base_data = nil
    end

    return true
  end
end
{'fraction_non_alphanumeric': 0.08699633699633699, 'fraction_numerical': 0.017704517704517704, 'mean_word_length': 2.9811664641555287, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3790346', 'n_tokens_mistral': 2271, 'n_tokens_neox': 2155, 'n_words': 532}