<commit_before>namespace :gemcutter do
  namespace :index do
    desc "Update the index"
    task update: :environment do
      require 'benchmark'
      Benchmark.bm do |b|
        b.report("update index") { Indexer.new.perform }
      end
    end
  end

  namespace :import do
    desc 'Bring the gems through the gemcutter process'
    task process: :environment do
      gems = Dir[File.join(ARGV[1] || "#{Gem.path.first}/cache", "*.gem")].sort.reverse
      puts "Processing #{gems.size} gems..."
      gems.each do |path|
        puts "Processing #{path}"
        cutter = Pusher.new(nil, File.open(path))

        cutter.process
        puts cutter.message unless cutter.code == 200
      end
    end
  end

  namespace :checksums do
    desc "Initialize missing checksums."
    task init: :environment do
      without_sha256 = Version.where(sha256: nil)
      mod = ENV['shard']
      without_sha256.where("id % 4 = ?", mod.to_i) if mod

      total = without_sha256.count
      i = 0
      without_sha256.find_each do |version|
        version.recalculate_sha256!
        i += 1
        print format("\r%.2f%% (%d/%d) complete", i.to_f / total * 100.0, i, total)
      end
      puts
      puts "Done."
    end

    desc "Check existing checksums."
    task check: :environment do
      failed = false
      Version.find_each do |version|
        actual_sha256 = version.recalculate_sha256
        if version.sha256 != actual_sha256
          puts "#{version.full_name}.gem has sha256 '#{actual_sha256}', " \
            "but '#{version.sha256}' was expected."
          failed = true
        end
      end

      exit 1 if failed
    end
  end

  namespace :rubygems do
    desc "Update the download counts for all gems."
    task update_download_counts: :environment do
      case_query = Rubygem
        .pluck(:name)
        .map { |name| "WHEN '#{name}' THEN #{Redis.current["downloads:rubygem:#{name}"].to_i}" }
        .join("\n            ")

      ActiveRecord::Base.connection.execute <<-SQL.strip_heredoc
        UPDATE rubygems
          SET downloads = CASE name
            #{case_query}
          END
      SQL
    end<commit_msg>Add rake task to update GemDownload counters<commit_after>namespace :gemcutter do
  namespace :index do
    desc "Update the index"
    task update: :environment do
      require 'benchmark'
      Benchmark.bm do |b|
        b.report("update index") { Indexer.new.perform }
      end
    end
  end

  namespace :import do
    desc 'Bring the gems through the gemcutter process'
    task process: :environment do
      gems = Dir[File.join(ARGV[1] || "#{Gem.path.first}/cache", "*.gem")].sort.reverse
      puts "Processing #{gems.size} gems..."
      gems.each do |path|
        puts "Processing #{path}"
        cutter = Pusher.new(nil, File.open(path))

        cutter.process
        puts cutter.message unless cutter.code == 200
      end
    end
  end

  namespace :checksums do
    desc "Initialize missing checksums."
    task init: :environment do
      without_sha256 = Version.where(sha256: nil)
      mod = ENV['shard']
      without_sha256.where("id % 4 = ?", mod.to_i) if mod

      total = without_sha256.count
      i = 0
      without_sha256.find_each do |version|
        version.recalculate_sha256!
        i += 1
        print format("\r%.2f%% (%d/%d) complete", i.to_f / total * 100.0, i, total)
      end
      puts
      puts "Done."
    end

    desc "Check existing checksums."
    task check: :environment do
      failed = false
      Version.find_each do |version|
        actual_sha256 = version.recalculate_sha256
        if version.sha256 != actual_sha256
          puts "#{version.full_name}.gem has sha256 '#{actual_sha256}', " \
            "but '#{version.sha256}' was expected."
          failed = true
        end
      end

      exit 1 if failed
    end
  end

  namespace :rubygems do
    desc "Update the db download count with the redis value"
    task update_download_counts: :environment do
      Version.find_in_batches do |group|
        sleep(2) # Make sure it doesn't get too crowded in there!
        group.each do |version|
          rubygem = version.rubygem
          begin
            GemDownload.create!(rubygem_id: version.rubygem_id, version_id: 0, count: rubygem.downloads || 0)
          rescue ActiveRecord::RecordNotUnique
            # Should not create same download entry
          end
          begin
            GemDownload.create!(rubygem_id: version.rubygem_id, version_id: version.id, count: version.downloads_count || 0)
          rescue ActiveRecord::RecordNotUnique
            # Should not create same download entry
          end
        end
      end
    end
{'fraction_non_alphanumeric': 0.0998719590268886, 'fraction_numerical': 0.023047375160051217, 'mean_word_length': 2.2015027322404372, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4193377', 'n_tokens_mistral': 1504, 'n_tokens_neox': 1405, 'n_words': 410}