<commit_before>require 'set'
require 'fileutils'
require 'application_utils'

class Repo
  attr_reader :logger, :grit_repo

  # This is the entry point to update the database from a recent pull.
  def self.update(path, *branches)
    new(path).update(branches)
  end

  def initialize(path)
    @logger    = Rails.logger
    @grit_repo = Grit::Repo.new(path)
  end

  def git_fetch
    git_exec('fetch', 'origin', '--quiet')
  end

  def git_show(sha1)
    git_exec('show', sha1)
  end

  # TODO: This method is getting long.
  def update(branches)
    ApplicationUtils.acquiring_lock_file('updating') do
      started_at = Time.now
      ncommits   = 0

      git_fetch
      pulled_at = Time.now

      if names_mapping_updated = names_mapping_updated?
        logger.info("there's a new names mapping")
      else
        logger.info("same names mapping than in last repo update")
      end

      Commit.transaction do
        ncommits = import_new_commits(branches)
        logger.info("#{ncommits} new commits imported into the database")

        break if ncommits.zero? && !names_mapping_updated

        only_for_new_commits = !names_mapping_updated
        contributor_names_per_commit = compute_contributor_names_per_commit(only_for_new_commits)
        manage_gone_contributors(contributor_names_per_commit.values.flatten.uniq) if names_mapping_updated

        logger.info("updating database")
        assign_contributors(contributor_names_per_commit)
        update_ranks
      end

      if cache_needs_expiration?(ncommits, names_mapping_updated)
        logger.info("expiring cache")
        expire_caches
      else
        logger.info("current cache is valid")
      end

      ended_at = Time.now

      logger.info("update completed in %.1f seconds" % [ended_at - started_at])
      RepoUpdate.create(
        :ncommits   => ncommits,
        :started_at => started_at,
        :pulled_at  => pulled_at,
        :ended_at   => ended_at
      )
    end
  end

protected

  # Determines whether the names mapping has been updated. This is useful because
  # if the names mapping is up to date we only need to assign contributors for
  # new commits.
  def names_mapping_updated?
    lastru = RepoUpdate.last
    # Use started_at in case a revised names manager is deployed while an update
    # is running.
    lastru ? NamesManager.mapping_updated_since?(lastru.started_at) : true
  end

  # Simple-minded git system caller, enough for what we need. Returns the output.
  def git_exec(command, *args)
    Dir.chdir(@grit_repo.working_dir) do
      %x{git #{command} #{args.join(' ')}}
    end
  end

  # Imports those commits in the Git repo that do not yet exist in the database.
  # To do that this method goes from HEAD downwards. New commits are imported
  # into the database, and as soon as a known commit comes up we are done.
  #
  # Note that commits are inserted in reverse order (most recent has lower ID)
  # and that order is not linear as new imports are performed, only relative to
  # commits within a given import.
  def import_new_commits(branches)
    ncommits = 0
    branches.each do |branch|
      ncommits += import_new_commits_in_branch(branch)
    end
    ncommits
  end

  # FIXME: This import is done this way for historical reasons but is
  # innefficient. We should figure out the last sha1 per branch, and
  # just directly pick up what's new with rev-list.
  def import_new_commits_in_branch(branch)
    batch_size   = 50
    ncommits     = 0
    offset       = 0
    three_months = 3.months.ago

    logger.info("importing new commits from branch #{branch}")
    loop do
      commits = @grit_repo.commits(branch, batch_size, offset)
      return ncommits if commits.empty?
      commits.each do |commit|
        if Commit.exists?(:sha1 => commit.id)
          if commit.committed_date < three_months
            # We need to stop at some point, so let's assume that no merge is
            # older than 3 months.
            return ncommits
          else
            # A merge can import commits older than existing ones, so just go on.
            next
          end
        end
        import_grit_commit(commit, branch)
        ncommits += 1
      end
      offset += commits.size
    end
  end

  # Creates a new commit from data in the given Grit commit object.
  def import_grit_commit(commit, branch)
    new_commit = Commit.new_from_grit_commit(commit, branch)
    # In very rare situations grit is not able to import some commits.
    # I have seen this just once, with messages like
    #
    #   unknown header 'gpgsig' in commit 31735bd39127784019893e4860b2c9807293da25
    #   unknown header '' in commit 31735bd39127784019893e4860b2c9807293da25
    #   ...
    #
    # That's why we check if new_commit is not nil.
    if new_commit && new_commit.save
      logger.info("imported commit #{new_commit.short_sha1}")
    else
      logger.error("couldn't import commit #{commit.id}")
      logger.error(new_commit.errors.full_messages)
    end
  end

  # Once the contributors are computed from the current commits, the ones in the
  # contributors table that are gone are destroyed. This happens when a new
  # equivalence is known for some contributor, when a "name" is added to the
  # black list in <tt>NamesManager.looks_like_an_author_name</tt>, etc.
  #
  # Contributions for the related commits will in general need to be updated,
  # we just clear them to ease this part, since diffing them by hand has some
  # cases to take into account and it is not worth the effort.
  def manage_gone_contributors(current_contributor_names)
    previous_contributor_names = NamesManager.all_names
    gone_names = previous_contributor_names - current_contributor_names<commit_msg>better a catch-all<commit_after>require 'set'
require 'fileutils'
require 'application_utils'

class Repo
  attr_reader :logger, :grit_repo

  # This is the entry point to update the database from a recent pull.
  def self.update(path, *branches)
    new(path).update(branches)
  end

  def initialize(path)
    @logger    = Rails.logger
    @grit_repo = Grit::Repo.new(path)
  end

  def git_fetch
    git_exec('fetch', 'origin', '--quiet')
  end

  def git_show(sha1)
    git_exec('show', sha1)
  end

  # TODO: This method is getting long.
  def update(branches)
    ApplicationUtils.acquiring_lock_file('updating') do
      started_at = Time.now
      ncommits   = 0

      git_fetch
      pulled_at = Time.now

      if names_mapping_updated = names_mapping_updated?
        logger.info("there's a new names mapping")
      else
        logger.info("same names mapping than in last repo update")
      end

      Commit.transaction do
        ncommits = import_new_commits(branches)
        logger.info("#{ncommits} new commits imported into the database")

        break if ncommits.zero? && !names_mapping_updated

        only_for_new_commits = !names_mapping_updated
        contributor_names_per_commit = compute_contributor_names_per_commit(only_for_new_commits)
        manage_gone_contributors(contributor_names_per_commit.values.flatten.uniq) if names_mapping_updated

        logger.info("updating database")
        assign_contributors(contributor_names_per_commit)
        update_ranks
      end

      if cache_needs_expiration?(ncommits, names_mapping_updated)
        logger.info("expiring cache")
        expire_caches
      else
        logger.info("current cache is valid")
      end

      ended_at = Time.now

      logger.info("update completed in %.1f seconds" % [ended_at - started_at])
      RepoUpdate.create(
        :ncommits   => ncommits,
        :started_at => started_at,
        :pulled_at  => pulled_at,
        :ended_at   => ended_at
      )
    end
  end

protected

  # Determines whether the names mapping has been updated. This is useful because
  # if the names mapping is up to date we only need to assign contributors for
  # new commits.
  def names_mapping_updated?
    lastru = RepoUpdate.last
    # Use started_at in case a revised names manager is deployed while an update
    # is running.
    lastru ? NamesManager.mapping_updated_since?(lastru.started_at) : true
  end

  # Simple-minded git system caller, enough for what we need. Returns the output.
  def git_exec(command, *args)
    Dir.chdir(@grit_repo.working_dir) do
      %x{git #{command} #{args.join(' ')}}
    end
  end

  # Imports those commits in the Git repo that do not yet exist in the database.
  # To do that this method goes from HEAD downwards. New commits are imported
  # into the database, and as soon as a known commit comes up we are done.
  #
  # Note that commits are inserted in reverse order (most recent has lower ID)
  # and that order is not linear as new imports are performed, only relative to
  # commits within a given import.
  def import_new_commits(branches)
    ncommits = 0
    branches.each do |branch|
      ncommits += import_new_commits_in_branch(branch)
    end
    ncommits
  end

  # FIXME: This import is done this way for historical reasons but is
  # innefficient. We should figure out the last sha1 per branch, and
  # just directly pick up what's new with rev-list.
  def import_new_commits_in_branch(branch)
    batch_size   = 50
    ncommits     = 0
    offset       = 0
    three_months = 3.months.ago

    logger.info("importing new commits from branch #{branch}")
    loop do
      commits = @grit_repo.commits(branch, batch_size, offset)
      return ncommits if commits.empty?
      commits.each do |commit|
        if Commit.exists?(:sha1 => commit.id)
          if commit.committed_date < three_months
            # We need to stop at some point, so let's assume that no merge is
            # older than 3 months.
            return ncommits
          else
            # A merge can import commits older than existing ones, so just go on.
            next
          end
        end
        import_grit_commit(commit, branch)
        ncommits += 1
      end
      offset += commits.size
    end
  end

  # Creates a new commit from data in the given Grit commit object.
  def import_grit_commit(commit, branch)
    new_commit = Commit.new_from_grit_commit(commit, branch)
    if new_commit.save
      logger.info("imported commit #{new_commit.short_sha1}")
    else
      logger.error("couldn't import commit #{commit.id}")
      logger.error(new_commit.errors.full_messages)
    end
  rescue
    # In very rare situations grit is not able to import some commits.
    # I have seen this just once, with messages like
    #
    #   unknown header 'gpgsig' in commit 31735bd39127784019893e4860b2c9807293da25
    #   unknown header '' in commit 31735bd39127784019893e4860b2c9807293da25
    #   ...
    #
    # That's why we have this catch-all.
    logger.error("couldn't import commit #{commit.id}")
    logger.error($!.message)
  end

  # Once the contributors are computed from the current commits, the ones in the
  # contributors table that are gone are destroyed. This happens when a new
  # equivalence is known for some contributor, when a "name" is added to the
  # black list in <tt>NamesManager.looks_like_an_author_name</tt>, etc.
  #
  # Contributions for the related commits will in general need to be updated,
  # we just clear them to ease this part, since diffing them by hand has some
  # cases to take into account and it is not worth the effort.
  def manage_gone_contributors(current_contributor_names)
    previous_contributor_names = NamesManager.all_names
    gone_names = previous_contributor_names - current_contributor_names
{'fraction_non_alphanumeric': 0.06748572417373248, 'fraction_numerical': 0.014016265789929053, 'mean_word_length': 2.9477459016393444, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4395263', 'n_tokens_mistral': 3607, 'n_tokens_neox': 3397, 'n_words': 1157}