<commit_before>require 'tempfile'

module Git
  
  class GitExecuteError < StandardError 
  end
  
  class Lib
      
    @git_dir = nil
    @git_index_file = nil
    @git_work_dir = nil
    @path = nil
    
    @logger = nil
    
    def initialize(base = nil, logger = nil)
      if base.is_a?(Git::Base)
        @git_dir = base.repo.path
        @git_index_file = base.index.path if base.index
        @git_work_dir = base.dir.path if base.dir
      elsif base.is_a?(Hash)
        @git_dir = base[:repository]
        @git_index_file = base[:index] 
        @git_work_dir = base[:working_directory]
      end
      @logger = logger
    end
    
    def init
      command('init')
    end
    
    # tries to clone the given repo
    #
    # returns {:repository} (if bare)
    #         {:working_directory} otherwise
    #
    # accepts options:
    #  :remote:: name of remote (rather than 'origin')
    #  :bare::   no working directory
    #  :depth::  the number of commits back to pull
    # 
    # TODO - make this work with SSH password or auth_key
    #
    def clone(repository, name, opts = {})
      @path = opts[:path] || '.'
      clone_dir = opts[:path] ? File.join(@path, name) : name
      
      arr_opts = []
      arr_opts << "--bare" if opts[:bare]
      arr_opts << "-o" << opts[:remote] if opts[:remote]
      arr_opts << "--depth" << opts[:depth].to_i if opts[:depth] && opts[:depth].to_i > 0

      arr_opts << '--'
      arr_opts << repository
      arr_opts << clone_dir
      
      command('clone', arr_opts)
      
      opts[:bare] ? {:repository => clone_dir} : {:working_directory => clone_dir}
    end
    
    
    ## READ COMMANDS ##
    
    
    def log_commits(opts = {})
      arr_opts = ['--pretty=oneline']
      arr_opts << "-#{opts[:count]}" if opts[:count]
      arr_opts << "--since=#{opts[:since]}" if opts[:since].is_a? String
      arr_opts << "--until=#{opts[:until]}" if opts[:until].is_a? String
      arr_opts << "--grep=#{opts[:grep]}" if opts[:grep].is_a? String
      arr_opts << "--author=#{opts[:author]}" if opts[:author].is_a? String
      arr_opts << "#{opts[:between][0].to_s}..#{opts[:between][1].to_s}" if (opts[:between] && opts[:between].size == 2)
      arr_opts << opts[:object] if opts[:object].is_a? String
      arr_opts << '--' << opts[:path_limiter] if opts[:path_limiter].is_a? String

      command_lines('log', arr_opts, true).map { |l| l.split.first }
    end
    
    def full_log_commits(opts = {})
      arr_opts = ['--pretty=raw']
      arr_opts << "-#{opts[:count]}" if opts[:count]
      arr_opts << "--skip=#{opts[:skip]}" if opts[:skip]
      arr_opts << "--since=#{opts[:since]}" if opts[:since].is_a? String
      arr_opts << "--until=#{opts[:until]}" if opts[:until].is_a? String
      arr_opts << "--grep=#{opts[:grep]}" if opts[:grep].is_a? String
      arr_opts << "--author=#{opts[:author]}" if opts[:author].is_a? String
      arr_opts << "#{opts[:between][0].to_s}..#{opts[:between][1].to_s}" if (opts[:between] && opts[:between].size == 2)
      arr_opts << opts[:object] if opts[:object].is_a? String
      arr_opts << '--' << opts[:path_limiter] if opts[:path_limiter].is_a? String
      
      full_log = command_lines('log', arr_opts, true)
      process_commit_data(full_log)
    end
    
    def revparse(string)
      return string if string =~ /[A-Fa-f0-9]{40}/  # passing in a sha - just no-op it
      rev = ['head', 'remotes', 'tags'].map do |d|
        File.join(@git_dir, 'refs', d, string)
      end.find do |path|
        File.file?(path)
      end
      return File.read(rev).chomp if rev
      command('rev-parse', string)
    end
    
    def namerev(string)
      command('name-rev', string).split[1]
    end
    
    def object_type(sha)
      command('cat-file', ['-t', sha])
    end
    
    def object_size(sha)
      command('cat-file', ['-s', sha]).to_i
    end
    
    # returns useful array of raw commit object data
    def commit_data(sha)
      sha = sha.to_s
      cdata = command_lines('cat-file', ['commit', sha])
      process_commit_data(cdata, sha)
    end
    
    def process_commit_data(data, sha = nil)
      in_message = false
            
      if sha
        hsh = {'sha' => sha, 'message' => '', 'parent' => []}
      else
        hsh_array = []        
      end
    
      data.each do |line|
        line = line.chomp
        if line == ''
          in_message = !in_message
        elsif in_message
          hsh['message'] << "\n" if hsh['message'] != ''
          hsh['message'] << line[4..-1]
        else
          data = line.split
          key = data.shift
          value = data.join(' ')
          if key == 'commit'
            sha = value
            hsh_array << hsh if hsh
            hsh = {'sha' => sha, 'message' => '', 'parent' => []}
          end
          if key == 'parent'
            hsh[key] << value
          else
            hsh[key] = value
          end
        end
      end
      
      if hsh_array
        hsh_array << hsh if hsh
        hsh_array
      else
        hsh<commit_msg>indent parameter for parsing commit data<commit_after>require 'tempfile'

module Git
  
  class GitExecuteError < StandardError 
  end
  
  class Lib
      
    @git_dir = nil
    @git_index_file = nil
    @git_work_dir = nil
    @path = nil
    
    @logger = nil
    
    def initialize(base = nil, logger = nil)
      if base.is_a?(Git::Base)
        @git_dir = base.repo.path
        @git_index_file = base.index.path if base.index
        @git_work_dir = base.dir.path if base.dir
      elsif base.is_a?(Hash)
        @git_dir = base[:repository]
        @git_index_file = base[:index] 
        @git_work_dir = base[:working_directory]
      end
      @logger = logger
    end
    
    def init
      command('init')
    end
    
    # tries to clone the given repo
    #
    # returns {:repository} (if bare)
    #         {:working_directory} otherwise
    #
    # accepts options:
    #  :remote:: name of remote (rather than 'origin')
    #  :bare::   no working directory
    #  :depth::  the number of commits back to pull
    # 
    # TODO - make this work with SSH password or auth_key
    #
    def clone(repository, name, opts = {})
      @path = opts[:path] || '.'
      clone_dir = opts[:path] ? File.join(@path, name) : name
      
      arr_opts = []
      arr_opts << "--bare" if opts[:bare]
      arr_opts << "-o" << opts[:remote] if opts[:remote]
      arr_opts << "--depth" << opts[:depth].to_i if opts[:depth] && opts[:depth].to_i > 0

      arr_opts << '--'
      arr_opts << repository
      arr_opts << clone_dir
      
      command('clone', arr_opts)
      
      opts[:bare] ? {:repository => clone_dir} : {:working_directory => clone_dir}
    end
    
    
    ## READ COMMANDS ##
    
    
    def log_commits(opts = {})
      arr_opts = ['--pretty=oneline']
      arr_opts << "-#{opts[:count]}" if opts[:count]
      arr_opts << "--since=#{opts[:since]}" if opts[:since].is_a? String
      arr_opts << "--until=#{opts[:until]}" if opts[:until].is_a? String
      arr_opts << "--grep=#{opts[:grep]}" if opts[:grep].is_a? String
      arr_opts << "--author=#{opts[:author]}" if opts[:author].is_a? String
      arr_opts << "#{opts[:between][0].to_s}..#{opts[:between][1].to_s}" if (opts[:between] && opts[:between].size == 2)
      arr_opts << opts[:object] if opts[:object].is_a? String
      arr_opts << '--' << opts[:path_limiter] if opts[:path_limiter].is_a? String

      command_lines('log', arr_opts, true).map { |l| l.split.first }
    end
    
    def full_log_commits(opts = {})
      arr_opts = ['--pretty=raw']
      arr_opts << "-#{opts[:count]}" if opts[:count]
      arr_opts << "--skip=#{opts[:skip]}" if opts[:skip]
      arr_opts << "--since=#{opts[:since]}" if opts[:since].is_a? String
      arr_opts << "--until=#{opts[:until]}" if opts[:until].is_a? String
      arr_opts << "--grep=#{opts[:grep]}" if opts[:grep].is_a? String
      arr_opts << "--author=#{opts[:author]}" if opts[:author].is_a? String
      arr_opts << "#{opts[:between][0].to_s}..#{opts[:between][1].to_s}" if (opts[:between] && opts[:between].size == 2)
      arr_opts << opts[:object] if opts[:object].is_a? String
      arr_opts << '--' << opts[:path_limiter] if opts[:path_limiter].is_a? String
      
      full_log = command_lines('log', arr_opts, true)
      process_commit_data(full_log)
    end
    
    def revparse(string)
      return string if string =~ /[A-Fa-f0-9]{40}/  # passing in a sha - just no-op it
      rev = ['head', 'remotes', 'tags'].map do |d|
        File.join(@git_dir, 'refs', d, string)
      end.find do |path|
        File.file?(path)
      end
      return File.read(rev).chomp if rev
      command('rev-parse', string)
    end
    
    def namerev(string)
      command('name-rev', string).split[1]
    end
    
    def object_type(sha)
      command('cat-file', ['-t', sha])
    end
    
    def object_size(sha)
      command('cat-file', ['-s', sha]).to_i
    end
    
    # returns useful array of raw commit object data
    def commit_data(sha)
      sha = sha.to_s
      cdata = command_lines('cat-file', ['commit', sha])
      process_commit_data(cdata, sha, 0)
    end
    
    def process_commit_data(data, sha = nil, indent = 4)
      in_message = false
            
      if sha
        hsh = {'sha' => sha, 'message' => '', 'parent' => []}
      else
        hsh_array = []        
      end
    
      data.each do |line|
        line = line.chomp
        if line == ''
          in_message = !in_message
        elsif in_message
          hsh['message'] << line[indent..-1] << "\n"
        else
          data = line.split
          key = data.shift
          value = data.join(' ')
          if key == 'commit'
            sha = value
            hsh_array << hsh if hsh
            hsh = {'sha' => sha, 'message' => '', 'parent' => []}
          end
          if key == 'parent'
            hsh[key] << value
          else
            hsh[key] = value
          end
        end
      end
      
      if hsh_array
        hsh_array << hsh if hsh
        hsh_array
      else
        hsh
{'fraction_non_alphanumeric': 0.15856550088915233, 'fraction_numerical': 0.0028650464335111636, 'mean_word_length': 2.1214924452667283, 'pattern_counts': {'":': 0, '<': 133, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7098946', 'n_tokens_mistral': 3406, 'n_tokens_neox': 3368, 'n_words': 782}