<commit_before>module InetData
  module Source
    class WhoisXMLAPI < Base

      def available?
        config['whoisxmlapi_username'].to_s.length > 0 &&
        config['whoisxmlapi_password'].to_s.length > 0
      end

      def download_index(prefix)
        target = URI.parse(config['whoisxmlapi_base_url'] + prefix)

        tries = 0
        begin

          tries += 1
          http = Net::HTTP.new(target.host, target.port)
          http.use_ssl = true if config['whoisxmlapi_base_url'].index("https") == 0

          req = Net::HTTP::Get.new(target.request_uri)
          req.basic_auth(config['whoisxmlapi_username'], config['whoisxmlapi_password'])

          res = http.request(req)

          unless (res and res.code.to_i == 200 and res.body.to_s.index("Index of /"))
            if res
              raise RuntimeError.new("Unexpected reply: #{res.code} - #{res['Content-Type']} - #{res.body.inspect}")
            else
              raise RuntimeError.new("Unexpected reply: #{res.code} - #{res['Content-Type']} - #{res.body.inspect}")
            end
          end

          res.body.to_s.scan(/a href=\"((\d+_\d+_\d+_|full_)[^\"]+)\"/m).map{|m| m.first}

        rescue ::Interrupt
          raise $!
        rescue ::Exception
          if tries < self.max_tries<commit_msg>Tweak to support a change at whoisxmlapi<commit_after>module InetData
  module Source
    class WhoisXMLAPI < Base

      def available?
        config['whoisxmlapi_username'].to_s.length > 0 &&
        config['whoisxmlapi_password'].to_s.length > 0
      end

      def download_index(prefix)
        target = URI.parse(config['whoisxmlapi_base_url'] + prefix)

        tries = 0
        begin

          tries += 1
          http = Net::HTTP.new(target.host, target.port)
          http.use_ssl = true if config['whoisxmlapi_base_url'].index("https") == 0

          req = Net::HTTP::Get.new(target.request_uri)
          req.basic_auth(config['whoisxmlapi_username'], config['whoisxmlapi_password'])

          res = http.request(req)

          unless (res and res.code.to_i == 200 and res.body.to_s.index("Last modified"))
            if res
              raise RuntimeError.new("Unexpected reply: #{res.code} - #{res['Content-Type']} - #{res.body.inspect}")
            else
              raise RuntimeError.new("Unexpected reply: #{res.code} - #{res['Content-Type']} - #{res.body.inspect}")
            end
          end

          res.body.to_s.scan(/a href=\"((\d+_\d+_\d+_|full_)[^\"]+)\"/m).map{|m| m.first}

        rescue ::Interrupt
          raise $!
        rescue ::Exception
          if tries < self.max_tries
{'fraction_non_alphanumeric': 0.14356435643564355, 'fraction_numerical': 0.006092916984006093, 'mean_word_length': 2.223312883435583, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5544659', 'n_tokens_mistral': 840, 'n_tokens_neox': 813, 'n_words': 153}