<commit_before>
# This is a one off. Written in order to reboot Velkomat data and start afresh.

require_relative './colorization'
require_relative './utility'

module Grove
  module CLI

    class IssueNuker

      include Colorization
      include Utility

      def self.build(program)
        program.command :dna_issue_nuker do |c|
          c.syntax "touch UID ..."
          c.description "Nuke DNA issues"
          c.action do |args, options|
            IssueNuker.new.nuke_all! 'post.issue'
            IssueNuker.new.nuke_all! 'post.issue_update'
          end
        end
      end


      def nuke_all!(klass, batch_size = 512)
        options = {
          realm: 'dna',
          klass: klass,
          deleted: false
        }
        ids = Post.unscoped.where(options).order('created_at DESC').pluck(:id)
        num = ids.count
        puts colorize("Total #{klass} to handle: #{num}", :red, :bright)

        count = 0
        percent = 0
        previous_message = 0
        ids.each_slice(batch_size) do |chunk|
          Post.find(chunk).each do |post|
            post.deleted = true
            post.save!
            count += 1
            message = "#{format("%.3f", (count * 100.0 / num))}%"
            if message != previous_message
              print "\b" * message.length
              print colorize(message, :yellow, :bright)
            end
            previous_message = message
          end<commit_msg>Reduce presicion on runtime output for the issue_nuker<commit_after>
# This is a one off. Written in order to reboot Velkomat data and start afresh.

require_relative './colorization'
require_relative './utility'

module Grove
  module CLI

    class IssueNuker

      include Colorization
      include Utility

      def self.build(program)
        program.command :dna_issue_nuker do |c|
          c.syntax "touch UID ..."
          c.description "Nuke DNA issues"
          c.action do |args, options|
            IssueNuker.new.nuke_all! 'post.issue'
            IssueNuker.new.nuke_all! 'post.issue_update'
          end
        end
      end


      def nuke_all!(klass, batch_size = 512)
        options = {
          realm: 'dna',
          klass: klass,
          deleted: false
        }
        ids = Post.unscoped.where(options).order('created_at DESC').pluck(:id)
        num = ids.count
        puts colorize("Total #{klass} to handle: #{num}", :red, :bright)

        count = 0
        percent = 0
        previous_message = 0
        ids.each_slice(batch_size) do |chunk|
          Post.find(chunk).each do |post|
            post.deleted = true
            post.save!
            count += 1
            message = "#{format("%.1f", (count * 100.0 / num))}%"
            if message != previous_message
              print "\b" * message.length
              print colorize(message, :yellow, :bright)
            end
            previous_message = message
          end
{'fraction_non_alphanumeric': 0.09907755380936112, 'fraction_numerical': 0.008199521694567818, 'mean_word_length': 1.7674858223062382, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1853270', 'n_tokens_mistral': 890, 'n_tokens_neox': 849, 'n_words': 234}