<commit_before>class HtmlSanitizer
  LINKABLE_URL_SCHEMES = URI.scheme_list.keys.map(&:downcase) - ['mailto'] + ['tel']

=begin

satinize html string based on whiltelist

  string = HtmlSanitizer.strict(string, external)

=end

  def self.strict(string, external = false)

    # config
    tags_remove_content = Rails.configuration.html_sanitizer_tags_remove_content
    tags_quote_content = Rails.configuration.html_sanitizer_tags_quote_content
    tags_whitelist = Rails.configuration.html_sanitizer_tags_whitelist
    attributes_whitelist = Rails.configuration.html_sanitizer_attributes_whitelist
    css_properties_whitelist = Rails.configuration.html_sanitizer_css_properties_whitelist
    css_values_blacklist = Rails.application.config.html_sanitizer_css_values_backlist

    # We whitelist yahoo_quoted because Yahoo Mail marks quoted email content using
    # <div class='yahoo_quoted'> and we rely on this class to identify quoted messages
    classes_whitelist = ['js-signatureMarker', 'yahoo_quoted']
    attributes_2_css = %w[width height]

    # remove html comments
    string.gsub!(/<!--.+?-->/m, '')

    scrubber_link = Loofah::Scrubber.new do |node|

      # wrap plain-text URLs in <a> tags
      if node.is_a?(Nokogiri::XML::Text) && node.ancestors.map(&:name).exclude?('a')
        urls = URI.extract(node.content, LINKABLE_URL_SCHEMES)
                  .map { |u| u.sub(/[,.]$/, '') }      # URI::extract captures trailing dots/commas
                  .reject { |u| u.match?(/^[^:]+:$/) } # URI::extract will match, e.g., 'tel:'

        next if urls.blank?

        add_link(node.content, urls, node)
      end

      # prepare links
      if node['href']
        href                = cleanup_target(node['href'], keep_spaces: true)
        href_without_spaces = href.gsub(/[[:space:]]/, '')
        if external && href_without_spaces.present? && !href_without_spaces.downcase.start_with?('//') && href_without_spaces.downcase !~ %r{^.{1,6}://.+?}
          node['href']        = "http://#{node['href']}"
          href                = node['href']
          href_without_spaces = href.gsub(/[[:space:]]/, '')
        end

        next if !href_without_spaces.downcase.start_with?('http', 'ftp', '//')

        node.set_attribute('href', href)
        node.set_attribute('rel', 'nofollow noreferrer noopener')
        node.set_attribute('target', '_blank')
      end<commit_msg>Improve performance by not checking empty strings or strings not containing ':' for URLs (issue #2374).<commit_after>class HtmlSanitizer
  LINKABLE_URL_SCHEMES = URI.scheme_list.keys.map(&:downcase) - ['mailto'] + ['tel']

=begin

satinize html string based on whiltelist

  string = HtmlSanitizer.strict(string, external)

=end

  def self.strict(string, external = false)

    # config
    tags_remove_content = Rails.configuration.html_sanitizer_tags_remove_content
    tags_quote_content = Rails.configuration.html_sanitizer_tags_quote_content
    tags_whitelist = Rails.configuration.html_sanitizer_tags_whitelist
    attributes_whitelist = Rails.configuration.html_sanitizer_attributes_whitelist
    css_properties_whitelist = Rails.configuration.html_sanitizer_css_properties_whitelist
    css_values_blacklist = Rails.application.config.html_sanitizer_css_values_backlist

    # We whitelist yahoo_quoted because Yahoo Mail marks quoted email content using
    # <div class='yahoo_quoted'> and we rely on this class to identify quoted messages
    classes_whitelist = ['js-signatureMarker', 'yahoo_quoted']
    attributes_2_css = %w[width height]

    # remove html comments
    string.gsub!(/<!--.+?-->/m, '')

    scrubber_link = Loofah::Scrubber.new do |node|

      # wrap plain-text URLs in <a> tags
      if node.is_a?(Nokogiri::XML::Text) && node.content.present? && node.content.include?(':') && node.ancestors.map(&:name).exclude?('a')
        urls = URI.extract(node.content, LINKABLE_URL_SCHEMES)
                  .map { |u| u.sub(/[,.]$/, '') }      # URI::extract captures trailing dots/commas
                  .reject { |u| u.match?(/^[^:]+:$/) } # URI::extract will match, e.g., 'tel:'

        next if urls.blank?

        add_link(node.content, urls, node)
      end

      # prepare links
      if node['href']
        href                = cleanup_target(node['href'], keep_spaces: true)
        href_without_spaces = href.gsub(/[[:space:]]/, '')
        if external && href_without_spaces.present? && !href_without_spaces.downcase.start_with?('//') && href_without_spaces.downcase !~ %r{^.{1,6}://.+?}
          node['href']        = "http://#{node['href']}"
          href                = node['href']
          href_without_spaces = href.gsub(/[[:space:]]/, '')
        end

        next if !href_without_spaces.downcase.start_with?('http', 'ftp', '//')

        node.set_attribute('href', href)
        node.set_attribute('rel', 'nofollow noreferrer noopener')
        node.set_attribute('target', '_blank')
      end
{'fraction_non_alphanumeric': 0.136776275973371, 'fraction_numerical': 0.0020173492031470646, 'mean_word_length': 3.2852203975799483, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2091739', 'n_tokens_mistral': 1655, 'n_tokens_neox': 1562, 'n_words': 297}