<commit_before># -*- coding: utf-8 -*-

require "zlib"

class Commit

  attr_accessor :oid, :parents, :msg, :committer, :author

  def initialize
    @parents = []
  end

  def to_hash
    {
      :oid => @oid,
      :parents => @parents,
      :msg => @msg,
      :committer => @committer,
      :author => @author
    }
  end

  def self.parse str
    c = Commit.new
    str.each_line{|line|
      case line
      when /^parent (.+)/    ; c.parents << $1
      when /^committer (.+)/ ; c.committer = $1
      when /^author (.+)/    ; c.author = $1
      end
    }

    msg = str.split("\n\n")[1]
    c.msg = msg
    c
  end

end

class Git

  def initialize dir
    @dir = File.expand_path(dir)
    @branches = []
    @commits = []
  end

  def read_file path
    File.read(File.join(@dir, ".git", path))
  end

  def read_file_bin path
    File.binread File.join(@dir, ".git", path)
  end

  def get_branches
    Dir.glob(@dir + "/.git/refs/heads/*").map{|path|
      File.basename(path)
    }
  end

  def to_hash
    {
      :dir => @dir,
      :branches => @branches,
      :commits => @commits,
      :head => @head
    }
  end

  def read_object_bin oid
    /^(..)(.+)$/ =~ oid
    index, name = $1, $2
    compressed = read_file_bin("objects/#{index}/#{name}")
    Zlib::Inflate.inflate compressed
  end

  def read_head_oid
    br_name = File.basename(read_file("HEAD").chomp)
    read_file("refs/heads/#{br_name}").chomp
  end
  
  def load_commit commits, oid
    if oid.nil? || commits.has_key?(oid)
      return commits
    end

    bin = read_object_bin(oid)
    /\A(.+?)\x00(.+)\Z/m =~ bin
    head, body = $1, $2

    c = Commit.parse(body)
    c.oid = oid
    commits[oid] = c
    
    commits = load_commit(commits, c.parents[0])
    commits = load_commit(commits, c.parents[1])
    
    commits
  end

  def load_commits br_name
    br_head_oid = read_file("refs/heads/#{br_name}").chomp
    load_commit({}, br_head_oid)
  end

  def load
    @branches = get_branches()
    commits_hash = {}
    @branches.each do |br_name|
      temp = load_commits(br_name)
      commits_hash.merge! temp
    end
    @commits = commits_hash.values.map{|commit|
      commit.to_hash
    }
    @head = read_head_oid()
  end

end


if __FILE__ == $0
  
  require "pp"

  dir = ARGV[0]
  git = Git.new(dir)
  git.load
  pp git.to_hash
end
<commit_msg>refactor<commit_after># -*- coding: utf-8 -*-

require "zlib"

class Commit

  attr_accessor :oid, :parents, :msg, :committer, :author

  def initialize
    @parents = []
  end

  def to_hash
    {
      :oid => @oid,
      :parents => @parents,
      :msg => @msg,
      :committer => @committer,
      :author => @author
    }
  end

  def self.parse str
    c = Commit.new
    str.each_line{|line|
      case line
      when /^parent (.+)/    ; c.parents << $1
      when /^committer (.+)/ ; c.committer = $1
      when /^author (.+)/    ; c.author = $1
      end
    }

    msg = str.split("\n\n")[1]
    c.msg = msg
    c
  end

end

class Git

  def initialize dir
    @dir = File.expand_path(dir)
    @branches = []
    @commits = []
  end

  def read_file path
    File.read(File.join(@dir, ".git", path))
  end

  def read_file_bin path
    File.binread File.join(@dir, ".git", path)
  end

  def get_branches
    Dir.glob(@dir + "/.git/refs/heads/*").map{|path|
      File.basename(path)
    }
  end

  def to_hash
    {
      :dir => @dir,
      :branches => @branches,
      :commits => @commits,
      :head => @head
    }
  end

  def read_object_bin oid
    /^(..)(.+)$/ =~ oid
    index, name = $1, $2
    compressed = read_file_bin("objects/#{index}/#{name}")
    Zlib::Inflate.inflate compressed
  end

  def read_head_oid
    br_name = File.basename(read_file("HEAD").chomp)
    read_file("refs/heads/#{br_name}").chomp
  end
  
  def load_commit commits, oid
    if oid.nil? || commits.has_key?(oid)
      return commits
    end

    bin = read_object_bin(oid)
    /\A(.+?)\x00(.+)\Z/m =~ bin
    head, body = $1, $2

    c = Commit.parse(body)
    c.oid = oid
    commits[oid] = c
    
    commits = load_commit(commits, c.parents[0])
    commits = load_commit(commits, c.parents[1])
    
    commits
  end

  def load_commits commits, br_name
    br_head_oid = read_file("refs/heads/#{br_name}").chomp
    load_commit(commits, br_head_oid)
  end

  def load
    @branches = get_branches()
    _commits = {}
    @branches.each do |br_name|
      _commits = load_commits(_commits, br_name)
    end
    @commits = _commits.values.map{|commit|
      commit.to_hash
    }
    @head = read_head_oid()
  end

end


if __FILE__ == $0
  
  require "pp"

  dir = ARGV[0]
  git = Git.new(dir)
  git.load
  pp git.to_hash
end

{'fraction_non_alphanumeric': 0.15017064846416384, 'fraction_numerical': 0.0063993174061433445, 'mean_word_length': 2.2248968363136177, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1471351', 'n_tokens_mistral': 1874, 'n_tokens_neox': 1799, 'n_words': 384}