<commit_before>module OrganicSitemap
  class UrlProcessor

    attr_accessor :status, :headers, :request, :env

    def initialize(status, headers, env)
      @request = Rack::Request.new env
      @status, @headers, @env = status, headers, env

    end

    def sanitize_path_info
      query_string = Rack::Utils.parse_nested_query(request.query_string)

      query_string.select!{|x,_| OrganicSitemap.configuration.allowed_params.include? x }

      sanitize_url = request.path
      sanitize_url << "?#{Rack::Utils.build_query(query_string.sort)}" if query_string.any?
      sanitize_url
    rescue => e
      p '*' * 100
      p "ERROR: " + e
      '/'
    end

    def sitemap_url?
      success_response? && html_page? && request.get? && is_expected_domain? && is_allowed_url?
    end

    private

    def success_response?
      status == 200
    end

    def html_page?
      headers["Content-Type"]["text/html"]
    end

    def is_expected_domain?
      # Any domain if not explicitly configured
      return true if OrganicSitemap.configuration.domains.nil?
      OrganicSitemap.configuration.domains.include? request.host
    end

    def is_allowed_url?
      # Any url if not explicitly configured
      return true unless OrganicSitemap.configuration.skipped_urls.any?
      [*OrganicSitemap.configuration.skipped_urls].each do |skip_url|
        return false if request.path[skip_url]
      end
      true
    end
  end
end<commit_msg>Improve allowed urls accept one RegExp<commit_after>module OrganicSitemap
  class UrlProcessor

    attr_accessor :status, :headers, :request, :env

    def initialize(status, headers, env)
      @request = Rack::Request.new env
      @status, @headers, @env = status, headers, env

    end

    def sanitize_path_info
      query_string = Rack::Utils.parse_nested_query(request.query_string)

      query_string.select!{|x,_| OrganicSitemap.configuration.allowed_params.include? x }

      sanitize_url = request.path
      sanitize_url << "?#{Rack::Utils.build_query(query_string.sort)}" if query_string.any?
      sanitize_url
    rescue => e
      p '*' * 100
      p "ERROR: " + e
      '/'
    end

    def sitemap_url?
      success_response? && html_page? && request.get? && is_expected_domain? && is_allowed_url?
    end

    private

    def success_response?
      status == 200
    end

    def html_page?
      headers["Content-Type"]["text/html"]
    end

    def is_expected_domain?
      # Any domain if not explicitly configured
      return true if OrganicSitemap.configuration.domains.nil?
      OrganicSitemap.configuration.domains.include? request.host
    end

    def is_allowed_url?
      # Any url if not explicitly configured
      return true unless OrganicSitemap.configuration.skipped_urls.present?
      [*OrganicSitemap.configuration.skipped_urls].each do |skip_url|
        return false if request.path[skip_url]
      end
      true
    end
  end
end
{'fraction_non_alphanumeric': 0.09056860742254001, 'fraction_numerical': 0.0040858018386108275, 'mean_word_length': 2.9702702702702704, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4211468', 'n_tokens_mistral': 942, 'n_tokens_neox': 904, 'n_words': 208}