<commit_before>require 'base64'

class Stresser

  APP_NAME = 'secret-plains-3232'

  def self.run
    loop do
      begin
        self.run_batch
      rescue StandardError => e
        puts "#{e.class}:#{e.message}\n#{e.backtrace.join("\n")}"
      rescue => e
        puts "unexpected error"
        puts "#{e.class}:#{e.message}\n#{e.backtrace.join("\n")}"
      else
        puts "batch completed"
      end
      sleep 1
    end
  end

  def self.random_letters
    2.times.map { SecureRandom.random_number(2**64).to_s(32) }.join('')
  end

  def self.random_url
    "postgres://#{random_letters}:#{random_letters}@example.com/#{random_letters}"
  end

  def self.test_bytea_encoding(db_url, len=32)
    Sequel.connect(db_url) do |c|
      actual = Sequel.blob(c.fetch("SELECT :bytes::bytea AS bytes",
                                   bytes: Sequel.blob(Random.new.bytes(len))).first[:bytes]).length
      if actual != len
        fail "Inconsistent bytea length: expected #{len}; got #{actual}"
      end
    end
  end

  def self.run_batch
    lock = Mutex.new
    active = true
    rand = Random.new
    sequel_thread = Thread.new do
      while lock.synchronize { active } do
        test_bytea_encoding(ENV['DATABASE_URL'])
      end
    end

    [ ByteaThing ].map do |klazz|
      Thread.new do
        obj = nil
        begin
          1000.times do
            obj = klazz.create
            obj.update(data: Sequel.blob(rand.bytes(176)),
                       stuff: random_url)
          end
        rescue StandardError => e
          bogus_value = obj.values[:stuff_encrypted]
          puts <<-EOF
bogus value is:
  class: #{bogus_value.class}
  length: #{bogus_value.length}
  raw value: #{Base64.encode64(bogus_value)}
EOF
          raise
        end
      end
    end.each(&:join)

    [ ByteaThing ].map do |klazz|
      klazz.dataset.delete
    end
    lock.synchronize { active = false }
  ensure
    sequel_thread.join unless sequel_thread.nil?
  end
end<commit_msg>Revert "Try removing 9.1 entirely instead"<commit_after>require 'base64'

class Stresser

  APP_NAME = 'secret-plains-3232'

  def self.run
    loop do
      begin
        self.run_batch
      rescue StandardError => e
        puts "#{e.class}:#{e.message}\n#{e.backtrace.join("\n")}"
      rescue => e
        puts "unexpected error"
        puts "#{e.class}:#{e.message}\n#{e.backtrace.join("\n")}"
      else
        puts "batch completed"
      end
      sleep 1
    end
  end

  def self.random_letters
    2.times.map { SecureRandom.random_number(2**64).to_s(32) }.join('')
  end

  def self.random_url
    "postgres://#{random_letters}:#{random_letters}@example.com/#{random_letters}"
  end

  def self.test_bytea_encoding(db_url, len=32)
    Sequel.connect(db_url) do |c|
      actual = Sequel.blob(c.fetch("SELECT :bytes::bytea AS bytes",
                                   bytes: Sequel.blob(Random.new.bytes(len))).first[:bytes]).length
      if actual != len
        fail "Inconsistent bytea length: expected #{len}; got #{actual}"
      end
    end
  end

  def self.run_batch
    lock = Mutex.new
    active = true
    rand = Random.new
    sequel_thread = Thread.new do
      while lock.synchronize { active } do
        test_bytea_encoding(ENV['NINE_ONE_DATABASE_URL'])
      end
    end

    [ NineOneByteaThing ].map do |klazz|
      Thread.new do
        obj = nil
        begin
          1000.times do
            obj = klazz.create
            obj.update(data: Sequel.blob(rand.bytes(176)),
                       stuff: random_url)
          end
        rescue StandardError => e
          bogus_value = obj.values[:stuff_encrypted]
          puts <<-EOF
bogus value is:
  class: #{bogus_value.class}
  length: #{bogus_value.length}
  raw value: #{Base64.encode64(bogus_value)}
EOF
          raise
        end
      end
    end.each(&:join)

    [ ByteaThing, NineOneByteaThing ].map do |klazz|
      klazz.dataset.delete
    end
    lock.synchronize { active = false }
  ensure
    sequel_thread.join unless sequel_thread.nil?
  end
end
{'fraction_non_alphanumeric': 0.11692155895411939, 'fraction_numerical': 0.013320177602368031, 'mean_word_length': 2.177899686520376, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1596760', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1349, 'n_words': 291}