<commit_before>require 'openssl'
require 'uri'

module HTML
  class Pipeline
    # HTML Filter for replacing http image URLs with camo versions. See:
    #
    # https://github.com/atmos/camo
    #
    # All images provided in user content should be run through this
    # filter so that http image sources do not cause mixed-content warnings
    # in browser clients.
    #
    # Context options:
    #   :asset_proxy (required) - Base URL for constructed asset proxy URLs.
    #   :asset_proxy_secret_key (required) - The shared secret used to encode URLs.
    #
    # This filter does not write additional information to the context.
    class CamoFilter < Filter
      # Hijacks images in the markup provided, replacing them with URLs that
      # go through the github asset proxy.
      def call
        doc.search("img").each do |element|
          next if context[:disable_asset_proxy]
          next if element['src'].nil?

          begin
            uri = URI.parse(element['src'])
          rescue Exception
            next
          end

          next if uri.hostname.nil?
          next if uri.hostname.match(/(^|\.)github(app)?\.com$/)

          element['src'] = asset_proxy_url(uri.to_s)
        end
        doc
      end

      # Implementation of validate hook.
      # Errors should raise exceptions or use an existing validator.
      def validate
        needs :asset_proxy, :asset_proxy_secret_key
      end

      # The camouflaged URL for a given image URL.
      def asset_proxy_url(url)
        "#{asset_proxy_host}/#{asset_url_hash(url)}/#{hexencode(url)}"
      end

      # Private: calculate the HMAC digest for a image source URL.
      def asset_url_hash(url)
        digest = OpenSSL::Digest::Digest.new('sha1')
        OpenSSL::HMAC.hexdigest(digest, asset_proxy_secret_key, url)
      end

      # Private: the hostname to use for generated asset proxied URLs.
      def asset_proxy_host
        context[:asset_proxy]
      end

      def asset_proxy_secret_key
        context[:asset_proxy_secret_key]
      end

      # Private: helper to hexencode a string. Each byte ends up encoded into
      # two characters, zero padded value in the range [0-9a-f].
      def hexencode(str)
        str.to_enum(:each_byte).map { |byte| "%02x" % byte }.join
      end
    end
  end
end<commit_msg>Fail fast if asset proxy is disabled<commit_after>require 'openssl'
require 'uri'

module HTML
  class Pipeline
    # HTML Filter for replacing http image URLs with camo versions. See:
    #
    # https://github.com/atmos/camo
    #
    # All images provided in user content should be run through this
    # filter so that http image sources do not cause mixed-content warnings
    # in browser clients.
    #
    # Context options:
    #   :asset_proxy (required) - Base URL for constructed asset proxy URLs.
    #   :asset_proxy_secret_key (required) - The shared secret used to encode URLs.
    #
    # This filter does not write additional information to the context.
    class CamoFilter < Filter
      # Hijacks images in the markup provided, replacing them with URLs that
      # go through the github asset proxy.
      def call
        return unless asset_proxy_enabled?

        doc.search("img").each do |element|
          next if element['src'].nil?

          begin
            uri = URI.parse(element['src'])
          rescue Exception
            next
          end

          next if uri.hostname.nil?
          next if uri.hostname.match(/(^|\.)github(app)?\.com$/)

          element['src'] = asset_proxy_url(uri.to_s)
        end
        doc
      end

      # Implementation of validate hook.
      # Errors should raise exceptions or use an existing validator.
      def validate
        needs :asset_proxy, :asset_proxy_secret_key
      end

      # The camouflaged URL for a given image URL.
      def asset_proxy_url(url)
        "#{asset_proxy_host}/#{asset_url_hash(url)}/#{hexencode(url)}"
      end

      # Private: calculate the HMAC digest for a image source URL.
      def asset_url_hash(url)
        digest = OpenSSL::Digest::Digest.new('sha1')
        OpenSSL::HMAC.hexdigest(digest, asset_proxy_secret_key, url)
      end

      # Private: Return true if asset proxy filter should be enabled
      def asset_proxy_enabled?
        !context[:disable_asset_proxy]
      end

      # Private: the hostname to use for generated asset proxied URLs.
      def asset_proxy_host
        context[:asset_proxy]
      end

      def asset_proxy_secret_key
        context[:asset_proxy_secret_key]
      end

      # Private: helper to hexencode a string. Each byte ends up encoded into
      # two characters, zero padded value in the range [0-9a-f].
      def hexencode(str)
        str.to_enum(:each_byte).map { |byte| "%02x" % byte }.join
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.07823199833990455, 'fraction_numerical': 0.002075119319360863, 'mean_word_length': 2.3425797503467405, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3691524', 'n_tokens_mistral': 1382, 'n_tokens_neox': 1343, 'n_words': 465}