<commit_before>namespace :populate do

  desc "Pull members from Github saves in database"
  task :users, [:repo] => :environment do |t, args|
    raise ArgumentError, "Must be called with repo argument. [:org/:repo]" if args[:repo].nil?
    repo = args[:repo]
    puts "Adding contributors to " + repo

    client = Hubstats::GithubAPI.client({:auto_paginate => true})
    contributors = client.contribs(args[:repo])

    contributors.each do |contributor|
      cont = Hubstats::User.create_or_update_user(contributor)
    end
  end

  desc "Pull comments from Github saves in database"
  task :comments, [:repo] => :environment do |t, args|
    raise ArgumentError, "Must be called with repo argument. [:org/:repo]" if args[:repo].nil?
    repo = Hubstats::Repo.where(full_name: args[:repo]).first
    puts "Adding comments to " + repo.full_name

    client = Hubstats::GithubAPI.client({:auto_paginate => true})

    pull_comments = Hubstats::GithubAPI.all(["repos",repo.full_name].join('/'),"pulls/comments")
    pull_comments.each do |comment|
      comm = Hubstats::Comment.create_or_update(comment_setup(comment,repo,"PullRequest"))
    end

    issue_comments = client.issues_comments(repo.full_name,{sort: 'created', direction: 'desc'})
    issue_comments.each do |comment|
      comm = Hubstats::Comment.create_or_update(comment_setup(comment,repo,"Issue"))
    end

    commit_comments = client.list_commit_comments(repo.full_name)
    commit_comments.each do |comment|
      comm = Hubstats::Comment.create_or_update(comment_setup(comment,repo,"Commit"))
    end
  end

  desc "Pull pull requests from Github saves in database"
  task :pulls, [:repo] => :environment do |t, args|
    raise ArgumentError, "Must be called with repo argument. [:org/:repo]" if args[:repo].nil?
    repo = Hubstats::Repo.where(full_name: args[:repo]).first
    puts "Adding pulls to " + repo.full_name

    client = Hubstats::GithubAPI.client({:auto_paginate => true})
    
    closed_pulls = client.pulls(repo.full_name, :state => "closed")
    pull_requests = closed_pulls.concat(client.pulls(repo.full_name, :state => "open"))
    pull_requests.each do |pull_request|
      pull = Hubstats::PullRequest.find_or_create_pull(pull_setup(pull_request))
    end
  end

  desc "Pull repos from Github save to database"
  task :all => :environment do
    client = Hubstats::GithubAPI.client({:auto_paginate => true})
    get_repos.each do |repo|
      re = Hubstats::Repo.create_or_update_repo(repo)

      Rake::Task["app:populate:users"].execute({repo: "#{re.full_name}"})
      Rake::Task["app:populate:pulls"].execute({repo: "#{re.full_name}"})
      Rake::Task["app:populate:comments"].execute({repo: "#{re.full_name}"})
      
    end
  end

  def get_repos
    client = Hubstats::GithubAPI.client
    if OCTOCONF.has_key?(:org_name)
      repos = client.organization_repositories(OCTOCONF[:org_name])
    else 
      repos = []
      OCTOCONF[:repo_list].each do |repo|
        repos << client.repository(repo)
      end
    end
    repos
  end

  def get_pull_number(comment)
    if comment[:pull_request]
      return comment[:pull_request][:number]
    elsif comment[:issue_url]
      return comment[:issue_url].split('/')[-1]
    elsif comment[:pull_request_url]
      return comment[:pull_request_url].split('/')[-1]
    else
      return nil
    end
  end

  def comment_setup(comment, repo, kind)
    comment[:repo_id] = repo.id
    comment[:pull_number] = get_pull_number(comment)
    comment[:kind] = kind
    return comment
  end

  def pull_setup(pull_request)
    pull_request[:repository] = pull_request[:head][:repo]
    return pull_request
  end
end<commit_msg>added hubstats pull-request updater<commit_after>namespace :populate do

  desc "Pull members from Github saves in database"
  task :users, [:repo] => :environment do |t, args|
    raise ArgumentError, "Must be called with repo argument. [:org/:repo]" if args[:repo].nil?
    repo = args[:repo]
    puts "Adding contributors to " + repo

    client = Hubstats::GithubAPI.client({:auto_paginate => true})
    contributors = client.contribs(args[:repo])

    contributors.each do |contributor|
      cont = Hubstats::User.create_or_update_user(contributor)
    end
  end

  desc "Pull comments from Github saves in database"
  task :comments, [:repo] => :environment do |t, args|
    raise ArgumentError, "Must be called with repo argument. [:org/:repo]" if args[:repo].nil?
    repo = Hubstats::Repo.where(full_name: args[:repo]).first
    puts "Adding comments to " + repo.full_name

    client = Hubstats::GithubAPI.client({:auto_paginate => true})

    pull_comments = Hubstats::GithubAPI.all(["repos",repo.full_name].join('/'),"pulls/comments")
    pull_comments.each do |comment|
      comm = Hubstats::Comment.create_or_update(comment_setup(comment,repo,"PullRequest"))
    end

    issue_comments = client.issues_comments(repo.full_name,{sort: 'created', direction: 'desc'})
    issue_comments.each do |comment|
      comm = Hubstats::Comment.create_or_update(comment_setup(comment,repo,"Issue"))
    end

    commit_comments = client.list_commit_comments(repo.full_name)
    commit_comments.each do |comment|
      comm = Hubstats::Comment.create_or_update(comment_setup(comment,repo,"Commit"))
    end
  end

  desc "Pull pull requests from Github saves in database"
  task :pulls, [:repo] => :environment do |t, args|
    raise ArgumentError, "Must be called with repo argument. [:org/:repo]" if args[:repo].nil?
    repo = Hubstats::Repo.where(full_name: args[:repo]).first
    puts "Adding pulls to " + repo.full_name

    client = Hubstats::GithubAPI.client({:auto_paginate => true})
    
    closed_pulls = client.pulls(repo.full_name, :state => "closed")
    pull_requests = closed_pulls.concat(client.pulls(repo.full_name, :state => "open"))
    pull_requests.each do |pull_request|
      pull = Hubstats::PullRequest.find_or_create_pull(pull_setup(pull_request))
    end
  end

  desc "Pull repos from Github save to database"
  task :all => :environment do
    client = Hubstats::GithubAPI.client({:auto_paginate => true})
    get_repos.each do |repo|
      re = Hubstats::Repo.create_or_update_repo(repo)

      Rake::Task["app:populate:users"].execute({repo: "#{re.full_name}"})
      Rake::Task["app:populate:pulls"].execute({repo: "#{re.full_name}"})
      Rake::Task["app:populate:comments"].execute({repo: "#{re.full_name}"})
    end
    puts "Finished with initial population, grabing extra info for pull requests"
    Rake::Task["app:populate:update"].execute()
  end

  desc "indivdually gets and updates pull requests"
  task :update => :environment do
    client = Hubstats::GithubAPI.client

    while Hubstats::PullRequest.where(deletions: nil).where(additions: nil).count() > 0
      incomplete = Hubstats::PullRequest.where(deletions: nil).where(additions: nil).limit(client.rate_limit.remaining)
      incomplete.each do |pull|
        repo = Hubstats::Repo.where(id: pull.repo_id).first
        pr = client.pull_request(repo.full_name, pull.number)

        Hubstats::PullRequest.find_or_create_pull(pull_setup(pr))
      end
      if Hubstats::PullRequest.where(deletions: nil).where(additions: nil).count() > 0
        puts "Hit Github rate limit, waiting to get more"
        sleep(5.minutes)
      end
    end
    
  end

  def get_repos
    client = Hubstats::GithubAPI.client
    if OCTOCONF.has_key?(:org_name)
      repos = client.organization_repositories(OCTOCONF[:org_name])
    else 
      repos = []
      OCTOCONF[:repo_list].each do |repo|
        repos << client.repository(repo)
      end
    end
    repos
  end

  def get_pull_number(comment)
    if comment[:pull_request]
      return comment[:pull_request][:number]
    elsif comment[:issue_url]
      return comment[:issue_url].split('/')[-1]
    elsif comment[:pull_request_url]
      return comment[:pull_request_url].split('/')[-1]
    else
      return nil
    end
  end

  def comment_setup(comment, repo, kind)
    comment[:repo_id] = repo.id
    comment[:pull_number] = get_pull_number(comment)
    comment[:kind] = kind
    return comment
  end

  def pull_setup(pull_request)
    pull_request[:repository] = pull_request[:base][:repo]
    return pull_request
  end
end
{'fraction_non_alphanumeric': 0.12901271956390067, 'fraction_numerical': 0.0008479709267110841, 'mean_word_length': 4.009708737864078, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363584', 'n_tokens_mistral': 2681, 'n_tokens_neox': 2611, 'n_words': 571}