<commit_before>started_at = Time.new.utc

API_URL = 'https://reestr.rublacklist.net/api/v2/ips/json'.freeze

Rails.logger.info('fetching blocked ips...')
content = RestClient.get(API_URL).body
ips = JSON.parse(content)
Rails.logger.info("total ips: #{ips.count}")

valid_ips = ips.select do |ip|
  begin
    IPAddr.new(ip)
    ip
  rescue IPAddr::InvalidAddressError => e
    Rails.logger.error(e)
  end
end

bad_ips_count = ips.length - valid_ips.length
Rails.logger.info("invalid ips: #{bad_ips_count}") if bad_ips_count.positive?

threshold = BlockedIP.order(updated_at: :desc).first.try(:updated_at)
prev_blocked_count = BlockedIP.count

values = valid_ips.map { |ip| "('#{ip}')" }.join(',')

query = "INSERT INTO blocked_ips (ip) VALUES #{values} " \
        "ON CONFLICT (ip) DO UPDATE SET updated_at = DEFAULT"

Rails.logger.info('updating db...')

begin
  ActiveRecord::Base.connection.exec_query(query)
rescue => e
  Rails.logger.error(e.message.to_s[0, 1000])
  exit
end

unblocked = BlockedIP.where('updated_at <= ?', threshold)
unblocked_count = unblocked.count

if unblocked_count.positive?
  Rails.logger.info("new unblocked ips: #{unblocked_count}")
  DataPoint.create_rkn_unblock(count: unblocked_count)<commit_msg>Prevent debug logging for the huge query<commit_after>started_at = Time.new.utc

API_URL = 'https://reestr.rublacklist.net/api/v2/ips/json'.freeze

Rails.logger.info('fetching blocked ips...')
content = RestClient.get(API_URL).body
ips = JSON.parse(content)
Rails.logger.info("total ips: #{ips.count}")

valid_ips = ips.select do |ip|
  begin
    IPAddr.new(ip)
    ip
  rescue IPAddr::InvalidAddressError => e
    Rails.logger.error(e)
  end
end

bad_ips_count = ips.length - valid_ips.length
Rails.logger.info("invalid ips: #{bad_ips_count}") if bad_ips_count.positive?

threshold = BlockedIP.order(updated_at: :desc).first.try(:updated_at)
prev_blocked_count = BlockedIP.count

values = valid_ips.map { |ip| "('#{ip}')" }.join(',')

query = "INSERT INTO blocked_ips (ip) VALUES #{values} " \
        "ON CONFLICT (ip) DO UPDATE SET updated_at = DEFAULT"

Rails.logger.info('updating db...')

ActiveRecord::Base.logger.silence(Logger::ERROR) do
  ActiveRecord::Base.connection.exec_query(query)
rescue => e
  Rails.logger.error(e.message)
  exit
end

unblocked = BlockedIP.where('updated_at <= ?', threshold)
unblocked_count = unblocked.count

if unblocked_count.positive?
  Rails.logger.info("new unblocked ips: #{unblocked_count}")
  DataPoint.create_rkn_unblock(count: unblocked_count)
{'fraction_non_alphanumeric': 0.14518048393494645, 'fraction_numerical': 0.002776675922253074, 'mean_word_length': 5.004761904761905, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4306823', 'n_tokens_mistral': 946, 'n_tokens_neox': 924, 'n_words': 171}