<commit_before>module Ci
  class JobTraceChunk < ActiveRecord::Base
    extend Gitlab::Ci::Model

    belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id

    after_destroy :redis_delete_data, if: :redis?

    default_value_for :data_store, :redis

    WriteError = Class.new(StandardError)

    CHUNK_SIZE = 128.kilobytes
    CHUNK_REDIS_TTL = 1.week
    LOCK_RETRY = 100
    LOCK_SLEEP = 1
    LOCK_TTL = 5.minutes

    enum data_store: {
      redis: 1,
      db: 2
    }

    def data
      if redis?
        redis_data
      elsif db?
        raw_data
      else
        raise 'Unsupported data store'
      end&.force_encoding(Encoding::BINARY)
    end

    def set_data(value)
      in_lock do
        raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE

        if redis?
          redis_set_data(value)
        elsif db?
          self.raw_data = value
        else
          raise 'Unsupported data store'
        end

        save! if changed?
        schedule_to_db if fullfilled?
      end
    end

    def truncate(offset = 0)
      self.append("", offset)
    end

    def append(new_data, offset)
      current_data = self.data.to_s
      raise ArgumentError, 'Offset is out of bound' if offset > current_data.bytesize || offset < 0
      raise ArgumentError, 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize<commit_msg>Put out schedule_to_db from inclock to avoid deadlock<commit_after>module Ci
  class JobTraceChunk < ActiveRecord::Base
    extend Gitlab::Ci::Model

    belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id

    after_destroy :redis_delete_data, if: :redis?

    default_value_for :data_store, :redis

    WriteError = Class.new(StandardError)

    CHUNK_SIZE = 128.kilobytes
    CHUNK_REDIS_TTL = 1.week
    LOCK_RETRY = 100
    LOCK_SLEEP = 1
    LOCK_TTL = 5.minutes

    enum data_store: {
      redis: 1,
      db: 2
    }

    def data
      if redis?
        redis_data
      elsif db?
        raw_data
      else
        raise 'Unsupported data store'
      end&.force_encoding(Encoding::BINARY)
    end

    def set_data(value)
      raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE

      in_lock do
        if redis?
          redis_set_data(value)
        elsif db?
          self.raw_data = value
        else
          raise 'Unsupported data store'
        end

        save! if changed?
      end

      schedule_to_db if fullfilled?
    end

    def truncate(offset = 0)
      self.append("", offset)
    end

    def append(new_data, offset)
      current_data = self.data.to_s
      raise ArgumentError, 'Offset is out of bound' if offset > current_data.bytesize || offset < 0
      raise ArgumentError, 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize
{'fraction_non_alphanumeric': 0.0735977134690961, 'fraction_numerical': 0.009289031797070382, 'mean_word_length': 2.3136094674556213, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6252427', 'n_tokens_mistral': 955, 'n_tokens_neox': 903, 'n_words': 244}