<commit_before>require 'cocaine'

module Kochiku
  module Worker
    class GitRepo
      class RefNotFoundError < StandardError; end
      WORKING_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'tmp', 'build-partition'))

      class << self
        def inside_copy(cached_repo_name, remote_name, repo_url, sha, branch)
          cached_repo_path = File.join(WORKING_DIR, cached_repo_name)
          synchronize_cache_repo(cached_repo_path, remote_name, repo_url, sha, branch)

          Dir.mktmpdir(nil, WORKING_DIR) do |dir|
            # clone local repo (fast!)
            run! "git clone #{cached_repo_path} #{dir}"

            Dir.chdir(dir) do
              raise RefNotFoundError.new("Build Ref #{sha} not found in #{repo_url}") unless system("git rev-list --quiet -n1 #{sha}")
              run! "git checkout --quiet #{sha}"

              run! "git submodule --quiet init"
              # redirect the submodules to the cached_repo
              submodules = `git config --get-regexp "^submodule\\..*\\.url$"`
              submodules.each_line do |config_line|
                submodule_path = config_line.match(/submodule\.(.*?)\.url/)[1]
                `git config --replace-all submodule.#{submodule_path}.url "#{cached_repo_path}/#{submodule_path}"`
              end

              run! "git submodule --quiet update"

              yield
            end
          end
        end

        def create_working_dir
          FileUtils.mkdir_p(WORKING_DIR)
        end

        private

        def synchronize_cache_repo(cached_repo_path, remote_name, repo_url, sha, branch)
          if !File.directory?(cached_repo_path)
            clone_repo(repo_url, cached_repo_path)
          end
          Dir.chdir(cached_repo_path) do
            remote_list = `git remote -v | grep #{remote_name}`
            unless remote_list.include?(remote_name)
              run! "git remote add #{remote_name} #{repo_url}"
            end

            synchronize_with_remote(remote_name, sha, branch)

	    # Update the master ref so that scripts may treat master build
	    # differently than branch build
            synchronize_with_remote(remote_name, sha, 'master') unless branch == 'master'

            Cocaine::CommandLine.new("git submodule update", "--init --quiet").run
          end
        end

        def run!(cmd)
          unless system(cmd)
            raise "non-0 exit code #{$?} returned from [#{cmd}]"
          end
        end

        def clone_repo(repo_url, cached_repo_path)
          Cocaine::CommandLine.new("git clone", "--recursive #{repo_url} #{cached_repo_path}").run
        end

        def synchronize_with_remote(name, sha, branch = nil)
          refspec = branch.to_s.empty? ? "" : "+#{branch}"
          Cocaine::CommandLine.new("git fetch", "--quiet --prune --no-tags #{name} #{refspec}").run
          # Check that we got the sha we are expecting
          Cocaine::CommandLine.new("git rev-list", "--quiet -n1 #{sha}").run
        rescue Cocaine::ExitStatusError => e
          # likely caused by another 'git fetch' that is currently in progress. Wait a few seconds and try again
          tries = (tries || 0) + 1
          if tries < 3
            Kochiku::Worker.logger.warn(e)
            sleep(15 * tries)
            retry<commit_msg>Mimic the submodule redirect logic in kochiku repo<commit_after>require 'cocaine'

module Kochiku
  module Worker
    class GitRepo
      class RefNotFoundError < StandardError; end
      WORKING_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'tmp', 'build-partition'))

      class << self
        def inside_copy(cached_repo_name, remote_name, repo_url, sha, branch)
          cached_repo_path = File.join(WORKING_DIR, cached_repo_name)
          synchronize_cache_repo(cached_repo_path, remote_name, repo_url, sha, branch)

          Dir.mktmpdir(nil, WORKING_DIR) do |dir|
            # clone local repo (fast!)
            run! "git clone #{cached_repo_path} #{dir}"

            Dir.chdir(dir) do
              raise RefNotFoundError.new("Build Ref #{sha} not found in #{repo_url}") unless system("git rev-list --quiet -n1 #{sha}")
              run! "git checkout --quiet #{sha}"

              run! "git submodule --quiet init"

              submodules = `git config --get-regexp "^submodule\\..*\\.url$"`

              unless submodules.empty?
                cached_submodules = nil
                inside_repo(cached_repo_path) do
                  cached_submodules = `git config --get-regexp "^submodule\\..*\\.url$"`
                end

                # Redirect the submodules to the cached_repo
                # If the submodule was added after the initial clone of the cache
                # repo then it will not be present in the cached_repo and we fall
                # back to cloning it for each build.
                submodules.each_line do |config_line|
                  if cached_submodules.include?(config_line)
                    submodule_path = config_line.match(/submodule\.(.*?)\.url/)[1]
                    `git config --replace-all submodule.#{submodule_path}.url "#{cached_repo_path}/#{submodule_path}"`
                  end
                end

                run! "git submodule --quiet update"
              end

              yield
            end
          end
        end

        def create_working_dir
          FileUtils.mkdir_p(WORKING_DIR)
        end

        private

        def inside_repo(cached_repo_path)
          Dir.chdir(cached_repo_path) do
            yield
          end
        end

        def synchronize_cache_repo(cached_repo_path, remote_name, repo_url, sha, branch)
          if !File.directory?(cached_repo_path)
            clone_repo(repo_url, cached_repo_path)
          end
          Dir.chdir(cached_repo_path) do
            remote_list = `git remote -v | grep #{remote_name}`
            unless remote_list.include?(remote_name)
              run! "git remote add #{remote_name} #{repo_url}"
            end

            synchronize_with_remote(remote_name, sha, branch)

            # Update the master ref so that scripts may treat master build
            # differently than branch build
            synchronize_with_remote(remote_name, sha, 'master') unless branch == 'master'

            Cocaine::CommandLine.new("git submodule update", "--init --quiet").run
          end
        end

        def run!(cmd)
          unless system(cmd)
            raise "non-0 exit code #{$?} returned from [#{cmd}]"
          end
        end

        def clone_repo(repo_url, cached_repo_path)
          Cocaine::CommandLine.new("git clone", "--recursive #{repo_url} #{cached_repo_path}").run
        end

        def synchronize_with_remote(name, sha, branch = nil)
          refspec = branch.to_s.empty? ? "" : "+#{branch}"
          Cocaine::CommandLine.new("git fetch", "--quiet --prune --no-tags #{name} #{refspec}").run
          # Check that we got the sha we are expecting
          Cocaine::CommandLine.new("git rev-list", "--quiet -n1 #{sha}").run
        rescue Cocaine::ExitStatusError => e
          # likely caused by another 'git fetch' that is currently in progress. Wait a few seconds and try again
          tries = (tries || 0) + 1
          if tries < 3
            Kochiku::Worker.logger.warn(e)
            sleep(15 * tries)
            retry
{'fraction_non_alphanumeric': 0.10546292530839095, 'fraction_numerical': 0.0024400162667751117, 'mean_word_length': 2.006519967400163, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1645409', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2069, 'n_words': 559}