<commit_before># Replace libc-based DNS resolution with pure Ruby DNS resolution, to avoid locking the ruby interpreter
require 'resolv-replace'

# Create Redis connection
redis_conn = proc {
  Redis.new url: Rails.application.secrets.redis_sidekiq
}

# Redis server location
Sidekiq.configure_server do |config|
  # Server needs (concurrency + 2) redis connections
  config.redis = ConnectionPool.new size: 5, &redis_conn
end

Sidekiq.configure_client do |config|
  # Client needs 1 redis connection per process (see Puma config, num of process = num of Puma workers)
  config.redis = ConnectionPool.new size: 1, &redis_conn
end

# Show error backtraces
Sidekiq.default_worker_options = { 'backtrace' => true }<commit_msg>Increment redis pool size for sidekiq.<commit_after># Replace libc-based DNS resolution with pure Ruby DNS resolution, to avoid locking the ruby interpreter
require 'resolv-replace'

# Create Redis connection
redis_conn = proc {
  Redis.new url: Rails.application.secrets.redis_sidekiq
}

# Redis server location
Sidekiq.configure_server do |config|
  # Server needs (concurrency + 2) redis connections
  config.redis = ConnectionPool.new size: 12, &redis_conn
end

Sidekiq.configure_client do |config|
  # Client needs 1 redis connection per process (see Puma config, num of process = num of Puma workers)
  config.redis = ConnectionPool.new size: 1, &redis_conn
end

# Show error backtraces
Sidekiq.default_worker_options = { 'backtrace' => true }
{'fraction_non_alphanumeric': 0.07540760869565218, 'fraction_numerical': 0.006114130434782609, 'mean_word_length': 5.061728395061729, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6176194', 'n_tokens_mistral': 471, 'n_tokens_neox': 445, 'n_words': 157}