<commit_before>require 'grit'

# This fix ensures sorted yaml maps.
class Hash
	def to_yaml( opts = {} )
		YAML::quick_emit( object_id, opts ) do |out|
      out.map( taguri, to_yaml_style ) do |map|
        sort_by { |k, v| k.to_s }.each do |k, v|
          map.add( k, v )
        end
      end
    end
	end
end

class GitStore

  class DefaultHandler
    def read(id, name, data)
      data
    end

    def write(data)
      data
    end
  end
  
  class YAMLHandler    
    def read(id, name, data)
      YAML.load(data)
    end

    def write(data)
      data.to_yaml
    end    
  end

  class RubyHandler
    def read(id, name, data)
      Object.module_eval(data)
    end
  end

  class ERBHandler
    def read(id, name, data)
      ERB.new(data)
    end
  end

  Handler = {
    'yml' => YAMLHandler.new,
    'rhtml' => ERBHandler.new,
    'rxml' => ERBHandler.new,
    'rb' => RubyHandler.new
  }

  Handler.default = DefaultHandler.new

  class Blob

    attr_reader :id
    attr_accessor :name

    def initialize(*args)
      if args.first.is_a?(Grit::Blob)
        @blob = args.first
        @id = @blob.id
        @name = @blob.name
      else
        @name = args[0]
        self.data = args[1]
      end
    end

    def extname
      File.extname(name)[1..-1]
    end

    def load(data)
      @data = handler.read(id, name, data)
    end

    def handler
      Handler[extname]
    end

    def data
      @data or (@blob and load(@blob.data))
    end

    def data=(data)
      @data = data
    end

    def to_s
      if handler.respond_to?(:write)
        handler.write(data)
      else
        @blob.data
      end
    end
    
  end

  class Tree
    include Enumerable

    attr_reader :id, :data
    attr_accessor :name
    
    def initialize(name = nil)
      @data = {}
      @name = name
    end

    def load(tree)
      @id = tree.id
      @name = tree.name
      @data = tree.contents.inject({}) do |hash, file|
        if file.is_a?(Grit::Tree)
          hash[file.name] = (@data[file.name] || Tree.new).load(file)
        else          
          hash[file.name] = Blob.new(file)
        end
        hash
      end
      self
    end

    def inspect
      "#<GitStore::Tree #{@data.inspect}>"
    end

    def fetch(name)
      name = name.to_s
      entry = @data[name]
      case entry
      when Blob then entry.data
      when Tree then entry<commit_msg>Removed the id argument from handlers.<commit_after>require 'grit'

# This fix ensures sorted yaml maps.
class Hash
	def to_yaml( opts = {} )
		YAML::quick_emit( object_id, opts ) do |out|
      out.map( taguri, to_yaml_style ) do |map|
        sort_by { |k, v| k.to_s }.each do |k, v|
          map.add( k, v )
        end
      end
    end
	end
end

class GitStore

  class DefaultHandler
    def read(name, data)
      data
    end

    def write(data)
      data
    end
  end
  
  class YAMLHandler    
    def read(name, data)
      YAML.load(data)
    end

    def write(data)
      data.to_yaml
    end    
  end

  class RubyHandler
    def read(name, data)
      Object.module_eval(data)
    end
  end

  class ERBHandler
    def read(name, data)
      ERB.new(data)
    end
  end

  Handler = {
    'yml' => YAMLHandler.new,
    'rhtml' => ERBHandler.new,
    'rxml' => ERBHandler.new,
    'rb' => RubyHandler.new
  }

  Handler.default = DefaultHandler.new

  class Blob

    attr_reader :id
    attr_accessor :name

    def initialize(*args)
      if args.first.is_a?(Grit::Blob)
        @blob = args.first
        @name = @blob.name
      else
        @name = args[0]
        self.data = args[1]
      end
    end

    def extname
      File.extname(name)[1..-1]
    end

    def load(data)
      @data = handler.read(name, data)
    end

    def handler
      Handler[extname]
    end

    def data
      @data or (@blob and load(@blob.data))
    end

    def data=(data)
      @data = data
    end

    def to_s
      if handler.respond_to?(:write)
        handler.write(data)
      else
        @blob.data
      end
    end
    
  end

  class Tree
    include Enumerable

    attr_reader :data
    attr_accessor :name
    
    def initialize(name = nil)
      @data = {}
      @name = name
    end

    def load(tree)
      @name = tree.name
      @data = tree.contents.inject({}) do |hash, file|
        if file.is_a?(Grit::Tree)
          hash[file.name] = (@data[file.name] || Tree.new).load(file)
        else          
          hash[file.name] = Blob.new(file)
        end
        hash
      end
      self
    end

    def inspect
      "#<GitStore::Tree #{@data.inspect}>"
    end

    def fetch(name)
      name = name.to_s
      entry = @data[name]
      case entry
      when Blob then entry.data
      when Tree then entry
{'fraction_non_alphanumeric': 0.1038550663576996, 'fraction_numerical': 0.0016852749104697704, 'mean_word_length': 1.7303047728579644, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5032490', 'n_tokens_mistral': 1696, 'n_tokens_neox': 1634, 'n_words': 430}