<commit_before># encoding: utf-8

namespace :dis do
  desc 'Check stuff'
  task consistency_check: :environment do
    unless ENV['MODELS']
      puts "Usage: #{$PROGRAM_NAME} dis:consistency_check " \
           'MODELS=Avatar,Document'
      exit
    end

    models = ENV['MODELS'].split(',').map(&:strip).map(&:constantize)

    models.each do |model|
      puts "-- #{model.name} --"

      content_hash_attr = model.dis_attributes[:content_hash]
      objects = model
                .select(content_hash_attr)
                .uniq
                .map { |r| r.send(content_hash_attr) }

      puts "Unique objects: #{objects.length}"

      Dis::Storage.layers.each do |layer|
        existing = objects
                   .pmap { |hash| [hash, layer.exists?(model.dis_type, hash)] }
                   .select(&:last)
                   .map(&:first)
        missing = objects - existing
        puts "#{layer.name}: #{existing.length} (#{missing.length} missing)"

        puts "Missing objects:\n#{missing.inspect}" if missing.any?
      end

      puts
    end
  end
end<commit_msg>Consistency check can now queue transfers to delayed layers<commit_after># encoding: utf-8

namespace :dis do
  desc 'Check stuff'
  task consistency_check: :environment do
    unless ENV['MODELS']
      puts "Usage: #{$PROGRAM_NAME} dis:consistency_check " \
           'MODELS=Avatar,Document'
      exit
    end

    models = ENV['MODELS'].split(',').map(&:strip).map(&:constantize)

    jobs = Set.new

    models.each do |model|
      puts "-- #{model.name} --"

      content_hash_attr = model.dis_attributes[:content_hash]
      objects = model
                .select(content_hash_attr)
                .uniq
                .map { |r| r.send(content_hash_attr) }
      global_missing = objects.dup

      puts "Unique objects: #{objects.length}"

      Dis::Storage.layers.each do |layer|
        print "Checking #{layer.name}... "
        existing = objects
                   .pmap { |hash| [hash, layer.exists?(model.dis_type, hash)] }
                   .select(&:last)
                   .map(&:first)
        missing = objects - existing
        global_missing -= existing
        puts "#{existing.length} existing, #{missing.length} missing"

        next unless layer.delayed? #&& layer.writeable?

        jobs += missing.pmap do |hash|
          [model.dis_type, hash] if Dis::Storage.exists?(model.dis_type, hash)
        end.compact
      end

      if global_missing.any?
        puts "\n#{global_missing.length} objects are missing from all layers:"
        pp global_missing
      end

      puts
    end

    if jobs.any?
      print "#{jobs.length} objects can be transferred to delayed layers, " \
            "queue now? (y/n) "
      response = STDIN.gets.chomp
      if response =~ /^y/i
        puts "Queueing jobs..."
        jobs.each { |type, hash| Dis::Jobs::Store.perform_later(type, hash) }
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.12580535774838927, 'fraction_numerical': 0.0006781959986436081, 'mean_word_length': 2.085774058577406, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2051613', 'n_tokens_mistral': 900, 'n_tokens_neox': 879, 'n_words': 211}