<commit_before>require 'typhoeus'
require 'uri'
require_relative './utils'

module HTML
  class Proofer
    class UrlValidator
      include HTML::Proofer::Utils

      attr_accessor :logger, :external_urls, :hydra

      def initialize(logger, external_urls, options, typhoeus_opts, hydra_opts)
        @logger = logger
        @external_urls = external_urls
        @failed_tests = []
        @options = options
        @hydra = Typhoeus::Hydra.new(hydra_opts)
        @typhoeus_opts = typhoeus_opts
      end

      def run
        external_link_checker(external_urls)
        @failed_tests
      end

      # Proofer runs faster if we pull out all the external URLs and run the checks
      # at the end. Otherwise, we're halting the consuming process for every file during
      # the check_directory_of_files process.
      #
      # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
      #
      # Finally, we'll first make a HEAD request, rather than GETing all the contents.
      # If the HEAD fails, we'll fall back to GET, as some servers are not configured
      # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
      # not an option.
      def external_link_checker(external_urls)
        external_urls = Hash[external_urls.sort]

        count = external_urls.length
        check_text = "#{count} " << (count == 1 ? 'external link' : 'external links')
        logger.log :info, :blue, "Checking #{check_text}..."

        Ethon.logger = logger # log from Typhoeus/Ethon

        url_processor(external_urls)

        logger.log :debug, :yellow, "Running requests for all #{hydra.queued_requests.size} external URLs..."
        hydra.run
      end

      def url_processor(external_urls)
        external_urls.each_pair do |href, filenames|
          href = clean_url(href)
          if hash?(href) && @options[:check_external_hash]
            queue_request(:get, href, filenames)
          else
            queue_request(:head, href, filenames)
          end
        end
      end

      def clean_url(href)
        Addressable::URI.parse(href).normalize
      end

      def queue_request(method, href, filenames)
        request = Typhoeus::Request.new(href, @typhoeus_opts.merge({ :method => method }))
        request.on_complete { |response| response_handler(response, filenames) }
        hydra.queue request
      end

      def response_handler(response, filenames)
        effective_url = response.options[:effective_url]
        href = response.request.base_url.to_s
        method = response.request.options[:method]
        response_code = response.code
        debug_msg = "Received a #{response_code} for #{href}"
        debug_msg << " in #{filenames.join(' ')}" unless filenames.nil?
        logger.log :debug, :yellow, debug_msg

        if response_code.between?(200, 299)
          check_hash_in_2xx_response(href, effective_url, response, filenames)
        elsif response.timed_out?
          handle_timeout(href, filenames, response_code)
        elsif method == :head
          queue_request(:get, href, filenames)
        else
          return if @options[:only_4xx] && !response_code.between?(400, 499)
          # Received a non-successful http response.
          add_failed_tests filenames, "External link #{href} failed: #{response_code} #{response.return_message}", response_code
        end
      end

      # Even though the response was a success, we may have been asked to check
      # if the hash on the URL exists on the page
      def check_hash_in_2xx_response(href, effective_url, response, filenames)
        return if @options[:only_4xx]
        return unless @options[:check_external_hash]
        return unless (hash = hash?(href))

        body_doc = create_nokogiri(response.body)

        # user-content is a special addition by GitHub.
        xpath = %(//*[@name="#{hash}"]|//*[@id="#{hash}"])
        if URI.parse(href).host.match(/github\.com/i)
          xpath << %(|//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])
        end

        return unless body_doc.xpath(xpath).empty?

        add_failed_tests filenames, "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not", response.code
      end

      def handle_timeout(href, filenames, response_code)
        return if @options[:only_4xx]
        add_failed_tests filenames, "External link #{href} failed: got a time out", response_code
      end

      def add_failed_tests(filenames, desc, status = nil)
        if filenames.nil?
          @failed_tests << CheckRunner::Issue.new('', desc, nil, status)
        else
          filenames.each { |f| @failed_tests << CheckRunner::Issue.new(f, desc, nil, status) }
        end<commit_msg>rename this method<commit_after>require 'typhoeus'
require 'uri'
require_relative './utils'

module HTML
  class Proofer
    class UrlValidator
      include HTML::Proofer::Utils

      attr_accessor :logger, :external_urls, :hydra

      def initialize(logger, external_urls, options, typhoeus_opts, hydra_opts)
        @logger = logger
        @external_urls = external_urls
        @failed_tests = []
        @options = options
        @hydra = Typhoeus::Hydra.new(hydra_opts)
        @typhoeus_opts = typhoeus_opts
      end

      def run
        external_link_checker(external_urls)
        @failed_tests
      end

      # Proofer runs faster if we pull out all the external URLs and run the checks
      # at the end. Otherwise, we're halting the consuming process for every file during
      # the check_directory_of_files process.
      #
      # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
      #
      # Finally, we'll first make a HEAD request, rather than GETing all the contents.
      # If the HEAD fails, we'll fall back to GET, as some servers are not configured
      # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
      # not an option.
      def external_link_checker(external_urls)
        external_urls = Hash[external_urls.sort]

        count = external_urls.length
        check_text = "#{count} " << (count == 1 ? 'external link' : 'external links')
        logger.log :info, :blue, "Checking #{check_text}..."

        Ethon.logger = logger # log from Typhoeus/Ethon

        url_processor(external_urls)

        logger.log :debug, :yellow, "Running requests for all #{hydra.queued_requests.size} external URLs..."
        hydra.run
      end

      def url_processor(external_urls)
        external_urls.each_pair do |href, filenames|
          href = clean_url(href)
          if hash?(href) && @options[:check_external_hash]
            queue_request(:get, href, filenames)
          else
            queue_request(:head, href, filenames)
          end
        end
      end

      def clean_url(href)
        Addressable::URI.parse(href).normalize
      end

      def queue_request(method, href, filenames)
        request = Typhoeus::Request.new(href, @typhoeus_opts.merge({ :method => method }))
        request.on_complete { |response| response_handler(response, filenames) }
        hydra.queue request
      end

      def response_handler(response, filenames)
        effective_url = response.options[:effective_url]
        href = response.request.base_url.to_s
        method = response.request.options[:method]
        response_code = response.code

        debug_msg = "Received a #{response_code} for #{href}"
        debug_msg << " in #{filenames.join(' ')}" unless filenames.nil?
        logger.log :debug, :yellow, debug_msg

        if response_code.between?(200, 299)
          check_hash_in_2xx_response(href, effective_url, response, filenames)
        elsif response.timed_out?
          handle_timeout(href, filenames, response_code)
        elsif method == :head
          queue_request(:get, href, filenames)
        else
          return if @options[:only_4xx] && !response_code.between?(400, 499)
          # Received a non-successful http response.
          add_external_issue(filenames, "External link #{href} failed: #{response_code} #{response.return_message}", response_code)
        end
      end

      # Even though the response was a success, we may have been asked to check
      # if the hash on the URL exists on the page
      def check_hash_in_2xx_response(href, effective_url, response, filenames)
        return if @options[:only_4xx]
        return unless @options[:check_external_hash]
        return unless (hash = hash?(href))

        body_doc = create_nokogiri(response.body)

        # user-content is a special addition by GitHub.
        xpath = %(//*[@name="#{hash}"]|//*[@id="#{hash}"])
        if URI.parse(href).host.match(/github\.com/i)
          xpath << %(|//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])
        end

        return unless body_doc.xpath(xpath).empty?

        add_external_issue filenames, "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not", response.code
      end

      def handle_timeout(href, filenames, response_code)
        return if @options[:only_4xx]
        add_external_issue filenames, "External link #{href} failed: got a time out", response_code
      end

      def add_external_issue(filenames, desc, status = nil)
        if filenames.nil?
          @failed_tests << CheckRunner::Issue.new('', desc, nil, status)
        else
          filenames.each { |f| @failed_tests << CheckRunner::Issue.new(f, desc, nil, status) }
        end
{'fraction_non_alphanumeric': 0.10143260483111995, 'fraction_numerical': 0.003764509045278678, 'mean_word_length': 2.6883918241419207, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1344222', 'n_tokens_mistral': 2886, 'n_tokens_neox': 2803, 'n_words': 780}