<commit_before># encoding: utf-8
require 'resolv'
require 'timeout'
require 'error_logger'

module DnsWorker

  include ErrorLogger

  def resolver(name)
    counter = 0

    begin
      Timeout.timeout(5) do
        results = Resolv.new.getaddress(name)
        return results
      end
    rescue Resolv::ResolvError
      return []
    rescue => error
      counter += 1
      sleep 1
      retry if counter < 3

      log_error("Problem encountered during a dns lookup for #{name}")
      log_error("DNS lookup returned: #{error}")

      return []
    end
  end

  def rbl_lookup(ip, query_host)
    reversed_ip = ip.split('.').reverse.join('.')
    response = resolver("#{reversed_ip}.#{query_host}")

    if response.empty?
      return response
    else
      return query_host
    end
  end

  def score_lookup(ip)
    reversed_ip = ip.split('.').reverse.join('.')
    response = resolver("#{reversed_ip}.score.senderscore.com")

    if response.empty?
      return 'no score'
    else
      return response.split('.')[3]
    end
  end
end
<commit_msg>more specific rbl query logic on responses<commit_after># encoding: utf-8
require 'resolv'
require 'timeout'
require 'error_logger'

module DnsWorker

  include ErrorLogger

  def resolver(name)
    counter = 0

    begin
      Timeout.timeout(5) do
        results = Resolv.new.getaddress(name)
        return results
      end
    rescue Resolv::ResolvError
      return []
    rescue => error
      counter += 1
      sleep 1
      retry if counter < 3

      log_error("Problem encountered during a dns lookup for #{name}")
      log_error("DNS lookup returned: #{error}")

      return []
    end
  end

  def rbl_lookup(ip, query_host)
    reversed_ip = ip.split('.').reverse.join('.')
    response = resolver("#{reversed_ip}.#{query_host}")

    if response.match(/^127\./)
      return query_host
    else
      return []
    end
  end

  def score_lookup(ip)
    reversed_ip = ip.split('.').reverse.join('.')
    response = resolver("#{reversed_ip}.score.senderscore.com")

    if response.match(/^127\./)
      return response.split('.')[3]
    else
      return 'no score'
    end
  end
end

{'fraction_non_alphanumeric': 0.11650485436893204, 'fraction_numerical': 0.009246417013407305, 'mean_word_length': 2.5533661740558293, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5477690', 'n_tokens_mistral': 706, 'n_tokens_neox': 693, 'n_words': 177}