<commit_before>class SeedRecord < ActiveRecord::Base
  belongs_to :seed_table
  scope :by_seed_table_id, ->(seed_table_id) { where(:seed_table_id => seed_table_id) }
  scope :by_record_id, ->(record_id) { where(:record_id => record_id) }
  scope :by_out_of_parts, ->(parts) {
    in_range = parts.map do |part|
      "(record_id BETWEEN #{part.record_id_from} AND #{part.record_id_to})"
    end.join(' OR ')
    where("NOT (#{in_range})")
  }

  BLOCK_SIZE = 5000

  class << self
    def renew_digests!(seed_express, inserted_ids, updated_ids, new_digests)
      pending_records = update_digests!(seed_express, updated_ids, new_digests)
      inserting_records = make_bulk_digest_records(seed_express, inserted_ids, new_digests)
      insert_digests!(seed_express, inserting_records + pending_records)
      delete_digests_out_of!(seed_express.parts)
    end

    private

    def update_digests!(seed_express, updated_ids, new_digests)
      inserting_records = []
      existing_digests = self.all.index_by(&:record_id)
      counter = 0
      seed_express.callbacks[:before_updating_digests].call(counter, updated_ids.size)
      updated_ids.each_slice(BLOCK_SIZE) do |part_of_updated_ids|
        seed_express.callbacks[:before_updating_a_part_of_digests].call(counter, updated_ids.size)
        updating_records = []
        part_of_updated_ids.each do |id|
          seed_record = existing_digests[id]
          if seed_record
            seed_record.digest = new_digests[id]
            updating_records << seed_record
          else
            inserting_records <<
              self.new(:record_id => id, :digest => new_digests[id])
          end
        end

        bulk_update_digests!(updating_records)
        counter += part_of_updated_ids.size
        seed_express.callbacks[:after_updating_a_part_of_digests].call(counter, updated_ids.size)
      end

      seed_express.callbacks[:after_updating_digests].call(counter, updated_ids.size)
      inserting_records
    end

    def make_bulk_digest_records(seed_express, inserted_ids, new_digests)
      counter = 0
      inserting_records = []
      seed_express.callbacks[:before_making_bulk_digest_records].call(0, inserted_ids.size)
      inserted_ids.each do |id|
        if counter % BLOCK_SIZE == 0
          seed_express.callbacks[:before_making_a_part_of_bulk_digest_records].call(counter, inserted_ids.size)
        end

        inserting_records <<
          self.new(:record_id => id, :digest => new_digests[id])

        counter += 1
        if counter % BLOCK_SIZE == 0
          seed_express.callbacks[:after_making_a_part_of_bulk_digest_records].call(counter, inserted_ids.size)
        end
      end

      seed_express.callbacks[:after_making_bulk_digest_records].call(inserted_ids.size, inserted_ids.size)
      inserting_records
    end

    def insert_digests!(seed_express, bulk_records)
      counter = 0
      bulk_records_count = bulk_records.size
      seed_express.callbacks[:before_inserting_digests].call(counter, bulk_records_count)
      while bulk_records.present?
        seed_express.callbacks[:before_inserting_a_part_of_digests].call(counter, bulk_records_count)
        targets = bulk_records.slice!(0, BLOCK_SIZE)
        SeedRecord.import(targets)
        counter += targets.size
        seed_express.callbacks[:after_inserting_a_part_of_digests].call(counter, bulk_records_count)
      end
      seed_express.callbacks[:after_inserting_digests].call(counter, bulk_records_count)
    end

    def bulk_update_digests!(records)
      return if records.empty?

      ids = records.map(&:id).join(',')
      digests = records.map { |v| "'#{v.digest}'"  }.join(',')
      updated_at = "'" + Time.zone.now.utc.strftime('%Y-%m-%dT%H:%M:%S') + "'"

      sql = <<-"EOF"
        UPDATE seed_records
        SET
          updated_at = #{updated_at},
          digest = ELT(FIELD(id, #{ids}), #{digests})
        WHERE
          id IN (#{ids})
      EOF

      ActiveRecord::Base.connection.execute(sql)
    end
<commit_msg>Uses Array#each_slice instead of Array#slice!<commit_after>class SeedRecord < ActiveRecord::Base
  belongs_to :seed_table
  scope :by_seed_table_id, ->(seed_table_id) { where(:seed_table_id => seed_table_id) }
  scope :by_record_id, ->(record_id) { where(:record_id => record_id) }
  scope :by_out_of_parts, ->(parts) {
    in_range = parts.map do |part|
      "(record_id BETWEEN #{part.record_id_from} AND #{part.record_id_to})"
    end.join(' OR ')
    where("NOT (#{in_range})")
  }

  BLOCK_SIZE = 5000

  class << self
    def renew_digests!(seed_express, inserted_ids, updated_ids, new_digests)
      pending_records = update_digests!(seed_express, updated_ids, new_digests)
      inserting_records = make_bulk_digest_records(seed_express, inserted_ids, new_digests)
      insert_digests!(seed_express, inserting_records + pending_records)
      delete_digests_out_of!(seed_express.parts)
    end

    private

    def update_digests!(seed_express, updated_ids, new_digests)
      inserting_records = []
      existing_digests = self.all.index_by(&:record_id)
      counter = 0
      seed_express.callbacks[:before_updating_digests].call(counter, updated_ids.size)
      updated_ids.each_slice(BLOCK_SIZE) do |part_of_updated_ids|
        seed_express.callbacks[:before_updating_a_part_of_digests].call(counter, updated_ids.size)
        updating_records = []
        part_of_updated_ids.each do |id|
          seed_record = existing_digests[id]
          if seed_record
            seed_record.digest = new_digests[id]
            updating_records << seed_record
          else
            inserting_records <<
              self.new(:record_id => id, :digest => new_digests[id])
          end
        end

        bulk_update_digests!(updating_records)
        counter += part_of_updated_ids.size
        seed_express.callbacks[:after_updating_a_part_of_digests].call(counter, updated_ids.size)
      end

      seed_express.callbacks[:after_updating_digests].call(counter, updated_ids.size)
      inserting_records
    end

    def make_bulk_digest_records(seed_express, inserted_ids, new_digests)
      counter = 0
      inserting_records = []
      seed_express.callbacks[:before_making_bulk_digest_records].call(0, inserted_ids.size)
      inserted_ids.each do |id|
        if counter % BLOCK_SIZE == 0
          seed_express.callbacks[:before_making_a_part_of_bulk_digest_records].call(counter, inserted_ids.size)
        end

        inserting_records <<
          self.new(:record_id => id, :digest => new_digests[id])

        counter += 1
        if counter % BLOCK_SIZE == 0
          seed_express.callbacks[:after_making_a_part_of_bulk_digest_records].call(counter, inserted_ids.size)
        end
      end

      seed_express.callbacks[:after_making_bulk_digest_records].call(inserted_ids.size, inserted_ids.size)
      inserting_records
    end

    def insert_digests!(seed_express, bulk_records)
      counter = 0
      bulk_records_count = bulk_records.size
      seed_express.callbacks[:before_inserting_digests].call(counter, bulk_records.size)
      bulk_records.each_slice(BLOCK_SIZE) do |targets|
        seed_express.callbacks[:before_inserting_a_part_of_digests].call(counter, bulk_records.size)
        SeedRecord.import(targets)
        seed_express.callbacks[:after_inserting_a_part_of_digests].call(counter += targets.size, bulk_records.size)
      end
      seed_express.callbacks[:after_inserting_digests].call(counter, bulk_records.size)
    end

    def bulk_update_digests!(records)
      return if records.empty?

      ids = records.map(&:id).join(',')
      digests = records.map { |v| "'#{v.digest}'"  }.join(',')
      updated_at = "'" + Time.zone.now.utc.strftime('%Y-%m-%dT%H:%M:%S') + "'"

      sql = <<-"EOF"
        UPDATE seed_records
        SET
          updated_at = #{updated_at},
          digest = ELT(FIELD(id, #{ids}), #{digests})
        WHERE
          id IN (#{ids})
      EOF

      ActiveRecord::Base.connection.execute(sql)
    end

{'fraction_non_alphanumeric': 0.10376652531803442, 'fraction_numerical': 0.0028685457720129706, 'mean_word_length': 3.377183406113537, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1846128', 'n_tokens_mistral': 2657, 'n_tokens_neox': 2573, 'n_words': 391}