<commit_before>module Mobilize
  class GitPath < Path
    include Mongoid::Document
    include Mongoid::Timestamps
    #a git_path points to a file inside a specific branch of a git repo
    field :service, type: String, default:->{"git"}
    field :domain, type: String, default:->{"github.com"}
    field :owner_name, type: String
    field :repo_name, type: String
    field :branch, type: String, default:->{"master"}
    field :file_path, type: String #file within the branch
    field :address, type: String, default:->{"#{domain}/#{owner_name}/#{repo_name}/#{branch}/#{file_path}"}
    field :url, type: String, default:->{ "#{service}://#{address}" } #unique identifier
    field :http_url, type: String, default:->{"https://#{domain}/#{owner_name}/#{repo_name}/blob/#{branch}/#{file_path}"}
    field :http_url_repo, type: String, default:->{"https://#{domain}/#{owner_name}/#{repo_name}.git"}
    field :git_user_name, type: String, default:->{"git"}
    field :ssh_url_repo, type: String, default:->{"#{git_user_name}@#{domain}:#{owner_name}/#{repo_name}.git"}

    validates :owner_name, :repo_name, presence: true

    #clones repo into temp folder with depth of 1
    #checks out appropriate branch
    def load(user=Mobilize.owner,run_dir=Dir.mktmpdir)
      gp = self
      begin
        #try http access first for public repo
        #use http access with nobody:nobody credentials
        gp.pub_load_cmd(run_dir).popen4(true)
      rescue
        #public access failed;
        #requires appropriate user permissions
        #given by private key
        gp.priv_load_cmd(run_dir,user).popen4(true)
      end
      repo_dir = "#{run_dir}/#{gp.repo_name}"
      return repo_dir
    end

    def pub_load_cmd(run_dir)
      cmd = "cd #{run_dir} && " +
            "git clone -q #{gp.http_url_repo.sub("https://","https://nobody:nobody@")} --depth=1 && " +
            "cd #{gp.repo_name} && git checkout #{gp.branch}"
      cmd
    end

    def priv_load_cmd(run_dir,user)
      key_value = user.git_key
      #create key file, set permissions, write key
      key_file_path = run_dir + "/key.ssh"
      File.open(key_file_path,"w") {|f| f.print(key_value)}
      "chmod 0600 #{key_file_path}".popen4
      #set git to not check strict host
      git_ssh_cmd = "#!/bin/sh\nexec /usr/bin/ssh -o 'UserKnownHostsFile=/dev/null' -o 'StrictHostKeyChecking=no' -i #{key_file_path} \"$@\""
      git_file_path = run_dir + "/git.ssh"
      File.open(git_file_path,"w") {|f| f.print(git_ssh_cmd)}
      "chmod 0700 #{git_file_path}".popen4
      #add keys, clone repo, go to specific revision, execute command
      cmd = "export GIT_SSH=#{git_file_path} && " +
            "cd #{run_dir} && " +
            "git clone -q #{gp.ssh_url_repo} --depth=1 && " +
            "cd #{gp.repo_name} && git checkout #{gp.branch}"
      return cmd
    end

    #loads repo and returns file contents as string
    def read(user=Mobilize.owner)
      gp = self
      repo_dir = gp.load
      File.read("#{repo_dir}/#{file_path}")
    end
  end
end<commit_msg>forgot to add gp<commit_after>module Mobilize
  class GitPath < Path
    include Mongoid::Document
    include Mongoid::Timestamps
    #a git_path points to a file inside a specific branch of a git repo
    field :service, type: String, default:->{"git"}
    field :domain, type: String, default:->{"github.com"}
    field :owner_name, type: String
    field :repo_name, type: String
    field :branch, type: String, default:->{"master"}
    field :file_path, type: String #file within the branch
    field :address, type: String, default:->{"#{domain}/#{owner_name}/#{repo_name}/#{branch}/#{file_path}"}
    field :url, type: String, default:->{ "#{service}://#{address}" } #unique identifier
    field :http_url, type: String, default:->{"https://#{domain}/#{owner_name}/#{repo_name}/blob/#{branch}/#{file_path}"}
    field :http_url_repo, type: String, default:->{"https://#{domain}/#{owner_name}/#{repo_name}.git"}
    field :git_user_name, type: String, default:->{"git"}
    field :ssh_url_repo, type: String, default:->{"#{git_user_name}@#{domain}:#{owner_name}/#{repo_name}.git"}

    validates :owner_name, :repo_name, presence: true

    #clones repo into temp folder with depth of 1
    #checks out appropriate branch
    def load(user=Mobilize.owner,run_dir=Dir.mktmpdir)
      gp = self
      begin
        #try http access first for public repo
        #use http access with nobody:nobody credentials
        gp.pub_load_cmd(run_dir).popen4(true)
      rescue
        #public access failed;
        #requires appropriate user permissions
        #given by private key
        gp.priv_load_cmd(run_dir,user).popen4(true)
      end
      repo_dir = "#{run_dir}/#{gp.repo_name}"
      return repo_dir
    end

    def pub_load_cmd(run_dir)
      gp = self
      cmd = "cd #{run_dir} && " +
            "git clone -q #{gp.http_url_repo.sub("https://","https://nobody:nobody@")} --depth=1 && " +
            "cd #{gp.repo_name} && git checkout #{gp.branch}"
      cmd
    end

    def priv_load_cmd(run_dir,user)
      gp = self
      key_value = user.git_key
      #create key file, set permissions, write key
      key_file_path = run_dir + "/key.ssh"
      File.open(key_file_path,"w") {|f| f.print(key_value)}
      "chmod 0600 #{key_file_path}".popen4
      #set git to not check strict host
      git_ssh_cmd = "#!/bin/sh\nexec /usr/bin/ssh -o 'UserKnownHostsFile=/dev/null' -o 'StrictHostKeyChecking=no' -i #{key_file_path} \"$@\""
      git_file_path = run_dir + "/git.ssh"
      File.open(git_file_path,"w") {|f| f.print(git_ssh_cmd)}
      "chmod 0700 #{git_file_path}".popen4
      #add keys, clone repo, go to specific revision, execute command
      cmd = "export GIT_SSH=#{git_file_path} && " +
            "cd #{run_dir} && " +
            "git clone -q #{gp.ssh_url_repo} --depth=1 && " +
            "cd #{gp.repo_name} && git checkout #{gp.branch}"
      return cmd
    end

    #loads repo and returns file contents as string
    def read(user=Mobilize.owner)
      gp = self
      repo_dir = gp.load
      File.read("#{repo_dir}/#{file_path}")
    end
  end
end
{'fraction_non_alphanumeric': 0.14906427990235965, 'fraction_numerical': 0.004882017900732303, 'mean_word_length': 3.0810092961487383, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7231866', 'n_tokens_mistral': 2110, 'n_tokens_neox': 2090, 'n_words': 508}