<commit_before># encoding: utf-8

class LinkFilter < Filter

  HTTPS_WHITELIST = [
    /\Ahttps?:\/\/([\w\d\-]+\.)?youtube\.(com|com\.br|fr|jp|nl|pl|es|ie|co\.uk)\//,
    /\Ahttps?:\/\/youtu\.be\//,
    /\Ahttps?:\/\/vimeo\.com\//,
    /\Ahttps?:\/\/soundcloud\.com\//
  ]

  def process(post)
    @post = post
    relativize_local_links!
    rewrite_for_https_support!
    parser.to_html
  end

  private

  def local_domains
    Sugar.config.domain_names.try(:strip).try(:split, /\s*,\s*/)
  end

  def parser
    @parser ||= Nokogiri::HTML::DocumentFragment.parse(@post)
  end

  def relativize_local_links!
    parser.search("a").each do |link|
      if href = link.try(:attributes).try(:[], 'href').try(:value)
        host = URI.parse(href).host
        if local_domains.find { |d| host == d }
          link.set_attribute 'href', href.gsub(Regexp.new("(https?:)?\/\/" + Regexp.escape(host)), "")
        end
      end
    end
  end

  def rewrite_for_https_support!
    parser.search("iframe").each do |iframe|
      if src = iframe.try(:attributes).try(:[], 'src').try(:value)
        if HTTPS_WHITELIST.find { |expr| src =~ expr }
          iframe.set_attribute 'src', src.gsub(/\Ahttps?:\/\//, "//")
        end
      end
    end
  end
end
<commit_msg>Use globbing for https match<commit_after># encoding: utf-8

class LinkFilter < Filter

  HTTPS_WHITELIST = %w{
    youtube.com
    *.youtube.com
    vimeo.com
    soundcloud.com
  }

  def process(post)
    @post = post
    relativize_local_links!
    rewrite_for_https_support!
    parser.to_html
  end

  private

  def local_domains
    Sugar.config.domain_names.try(:strip).try(:split, /\s*,\s*/)
  end

  def parser
    @parser ||= Nokogiri::HTML::DocumentFragment.parse(@post)
  end

  def relativize_local_links!
    parser.search("a").each do |link|
      if href = link.try(:attributes).try(:[], 'href').try(:value)
        host = URI.parse(href).host
        if local_domains.find { |d| host == d }
          link.set_attribute 'href', href.gsub(Regexp.new("(https?:)?\/\/" + Regexp.escape(host)), "")
        end
      end
    end
  end

  def rewrite_for_https_support!
    parser.search("iframe").each do |iframe|
      if src = iframe.try(:attributes).try(:[], 'src').try(:value)
        host = URI.parse(src).host
        if HTTPS_WHITELIST.find { |domain| File.fnmatch(domain, host) }
          iframe.set_attribute 'src', src.gsub(/\Ahttps?:\/\//, "//")
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.17502021018593372, 'fraction_numerical': 0.0008084074373484236, 'mean_word_length': 2.5306704707560628, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2613101', 'n_tokens_mistral': 927, 'n_tokens_neox': 889, 'n_words': 159}