<commit_before>require "tempfile"
require "thread"

class RepositoryTask
  include Rake::DSL

  class ThreadPool
    def initialize(n_workers=nil, &worker)
      @n_workers = n_workers || detect_n_processors
      @worker = worker
      @jobs = Thread::Queue.new
      @workers = @n_workers.times.collect do
        Thread.new do
          loop do
            job = @jobs.pop
            break if job.nil?
            @worker.call(job)
          end
        end
      end
    end

    def <<(job)
      @jobs << job
    end

    def join
      @n_workers.times do
        @jobs << nil
      end
      @workers.each(&:join)
    end

    private
    def detect_n_processors
      if File.exist?("/proc/cpuinfo")
        File.readlines("/proc/cpuinfo").grep(/^processor/).size
      else
        8
      end
    end
  end

  def define
    define_repository_task
    define_gpg_task
    define_yum_task
    define_apt_task
  end

  private
  def env_value(name)
    value = ENV[name]
    raise "Specify #{name} environment variable" if value.nil?
    value
  end

  def primary_gpg_uid
    gpg_uids.first
  end

  def shorten_gpg_uid(uid)
    uid[-8..-1]
  end

  def gpg_key_path(uid)
    "GPG-KEY-#{shorten_gpg_uid(uid).downcase}"
  end

  def rpm_gpg_key_path(uid)
    "RPM-GPG-KEY-#{shorten_gpg_uid(uid).downcase}"
  end

  def repositories_dir
    "repositories"
  end

  def define_repository_task
    directory repositories_dir
  end

  def define_gpg_task
    paths = []
    gpg_uids.each do |gpg_uid|
      path = gpg_key_path(gpg_uid)
      paths << path
      file path do |task|
        uid = gpg_uid
        unless system("gpg", "--list-keys", uid, :out => IO::NULL)
          sh("gpg",
             "--keyserver", "keyserver.ubuntu.com",
             "--recv-key", uid)
        end
        sh("gpg", "--armor", "--export", uid, :out => path)
      end
    end
  end

  def rpm_version(spec)
    content = File.read(spec)
    version = content.scan(/^Version: (.+)$/)[0][0]
    release = content.scan(/^Release: (.+)$/)[0][0]
    "#{version}-#{release}"
  end
<commit_msg>Remove unused variable<commit_after>require "tempfile"
require "thread"

class RepositoryTask
  include Rake::DSL

  class ThreadPool
    def initialize(n_workers=nil, &worker)
      @n_workers = n_workers || detect_n_processors
      @worker = worker
      @jobs = Thread::Queue.new
      @workers = @n_workers.times.collect do
        Thread.new do
          loop do
            job = @jobs.pop
            break if job.nil?
            @worker.call(job)
          end
        end
      end
    end

    def <<(job)
      @jobs << job
    end

    def join
      @n_workers.times do
        @jobs << nil
      end
      @workers.each(&:join)
    end

    private
    def detect_n_processors
      if File.exist?("/proc/cpuinfo")
        File.readlines("/proc/cpuinfo").grep(/^processor/).size
      else
        8
      end
    end
  end

  def define
    define_repository_task
    define_gpg_task
    define_yum_task
    define_apt_task
  end

  private
  def env_value(name)
    value = ENV[name]
    raise "Specify #{name} environment variable" if value.nil?
    value
  end

  def primary_gpg_uid
    gpg_uids.first
  end

  def shorten_gpg_uid(uid)
    uid[-8..-1]
  end

  def gpg_key_path(uid)
    "GPG-KEY-#{shorten_gpg_uid(uid).downcase}"
  end

  def rpm_gpg_key_path(uid)
    "RPM-GPG-KEY-#{shorten_gpg_uid(uid).downcase}"
  end

  def repositories_dir
    "repositories"
  end

  def define_repository_task
    directory repositories_dir
  end

  def define_gpg_task
    gpg_uids.each do |gpg_uid|
      path = gpg_key_path(gpg_uid)
      file path do |task|
        uid = gpg_uid
        unless system("gpg", "--list-keys", uid, :out => IO::NULL)
          sh("gpg",
             "--keyserver", "keyserver.ubuntu.com",
             "--recv-key", uid)
        end
        sh("gpg", "--armor", "--export", uid, :out => path)
      end
    end
  end

  def rpm_version(spec)
    content = File.read(spec)
    version = content.scan(/^Version: (.+)$/)[0][0]
    release = content.scan(/^Release: (.+)$/)[0][0]
    "#{version}-#{release}"
  end

{'fraction_non_alphanumeric': 0.11865635572740454, 'fraction_numerical': 0.0033832769453842437, 'mean_word_length': 2.1096919609316305, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2179875', 'n_tokens_mistral': 1487, 'n_tokens_neox': 1434, 'n_words': 313}