<commit_before>class Repository < ActiveRecord::Base
  after_save      :fetch_commits_from_github
  has_many        :commits, dependent: :destroy
  validates       :url, presence: true, uniqueness: true
  attr_accessible :url
  attr_reader :path

  def as_json options = {}
    options.reverse_merge! include: :commits
    super options
  end

  def fetch_commits_from_github
    github.commits(path).each do |commit|
      commits.create sha: commit.sha,
               timestamp: commit.commit.author.date
    end
  end

  def github
    return @github if @github

    @github = Octokit::Client.new
    
    if host != 'github.com'
      @github.api_endpoint = "https://#{host}/api/v3"
      @github.web_endpoint = "https://#{host}/"
    end

    @github
  end

  def host
    @host || (@host, @path = url.split '/', 2)
    @host
  end

  def path
    @path || (@host, @path = url.split '/', 2)
    @path
  end

  def url= _url
    self[:url] = Repository.clean_url(_url)
  end

  def self.with_url _url
    where(url: clean_url(_url)).first_or_create
  end

  def self.clean_url _url
    _url.sub(/^[^\/]*(?:\/\/|@)/,'').sub(/\.git/,'')
  end
end
<commit_msg>use committer date for commit timestamps<commit_after>class Repository < ActiveRecord::Base
  after_save      :fetch_commits_from_github
  has_many        :commits, dependent: :destroy
  validates       :url, presence: true, uniqueness: true
  attr_accessible :url
  attr_reader :path

  def as_json options = {}
    options.reverse_merge! include: :commits
    super options
  end

  def fetch_commits_from_github
    github.commits(path).each do |commit|
      commits.create sha: commit.sha,
               timestamp: commit.commit.committer.date
    end
  end

  def github
    return @github if @github

    @github = Octokit::Client.new
    
    if host != 'github.com'
      @github.api_endpoint = "https://#{host}/api/v3"
      @github.web_endpoint = "https://#{host}/"
    end

    @github
  end

  def host
    @host || (@host, @path = url.split '/', 2)
    @host
  end

  def path
    @path || (@host, @path = url.split '/', 2)
    @path
  end

  def url= _url
    self[:url] = Repository.clean_url(_url)
  end

  def self.with_url _url
    where(url: clean_url(_url)).first_or_create
  end

  def self.clean_url _url
    _url.sub(/^[^\/]*(?:\/\/|@)/,'').sub(/\.git/,'')
  end
end

{'fraction_non_alphanumeric': 0.13423959218351741, 'fraction_numerical': 0.002548853016142736, 'mean_word_length': 2.634259259259259, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1745535', 'n_tokens_mistral': 840, 'n_tokens_neox': 809, 'n_words': 193}