<commit_before>require "bentley_mcilroy"

require "vcdiff/integer"
require "vcdiff/records"
require "vcdiff/code_table"

module VCDIFF
  class Encoder
    #### Implement me ####
  end

  class Decoder
    attr_accessor :dictionary

    def initialize(dictionary)
      @dictionary = File.read(dictionary)

      @s_near = 4
      @s_same = 3

      # all cache values initialize to 0
      @near_cache = Array.new(@s_near, 0)
      @same_cache = Array.new(@s_same * 256, 0)
      @next_slot = 0
    end

    # Decodes a delta file using the dictionary given to the decoder
    def decode(file)
      delta_file = DeltaFile.read(File.new(file))

      if delta_file.header.header_indicator != 0
        raise NotImplementedError, "Header indicator of #{delta_file.header.header_indicator} can't be handled"
      end

      if delta_file.header.custom_codetable?
        raise NotImplementedError, "Unable to handle a custom codetable"
      end

      # there's no simple way to determine the number of windows, since the
      # count isn't given ahead of time, so we'll need to manually iterate
      # through the windows
      window_stream = StringIO.new(delta_file.windows)

      target_file = ""
<commit_msg>Force the source file to be read as binary, ASCII-8BIT, so it plays nicely with the delta file when dealing with diffs of binary data files<commit_after>require "bentley_mcilroy"

require "vcdiff/integer"
require "vcdiff/records"
require "vcdiff/code_table"

module VCDIFF
  class Encoder
    #### Implement me ####
  end

  class Decoder
    attr_accessor :dictionary

    def initialize(dictionary)
      @dictionary = File.read(dictionary, mode: "rb")

      @s_near = 4
      @s_same = 3

      # all cache values initialize to 0
      @near_cache = Array.new(@s_near, 0)
      @same_cache = Array.new(@s_same * 256, 0)
      @next_slot = 0
    end

    # Decodes a delta file using the dictionary given to the decoder
    def decode(file)
      delta_file = DeltaFile.read(File.new(file))

      if delta_file.header.header_indicator != 0
        raise NotImplementedError, "Header indicator of #{delta_file.header.header_indicator} can't be handled"
      end

      if delta_file.header.custom_codetable?
        raise NotImplementedError, "Unable to handle a custom codetable"
      end

      # there's no simple way to determine the number of windows, since the
      # count isn't given ahead of time, so we'll need to manually iterate
      # through the windows
      window_stream = StringIO.new(delta_file.windows)

      target_file = ""

{'fraction_non_alphanumeric': 0.07821011673151751, 'fraction_numerical': 0.008171206225680934, 'mean_word_length': 3.010920436817473, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3983531', 'n_tokens_mistral': 823, 'n_tokens_neox': 752, 'n_words': 251}