<commit_before>module OEmbed
  class Provider
    class NotFound<StandardError;end
    class UnknownFormat<StandardError;end
    
    attr_accessor :format, :name, :url, :urls, :endpoint
    
    def initialize(endpoint, format = :json)
      @endpoint = endpoint
      @urls = []
      @format = :json
    end
    
    def <<(url)
      full, scheme, domain, path = *url.match(%r{([^:]*)://?([^/?]*)(.*)})
      domain = Regexp.escape(domain).gsub("\\*", "(.*?)").gsub("(.*?)\\.", "([^\\.]+\\.)?")
      path.gsub!("*", "(.*?)")
      @urls << Regexp.new("^#{scheme}://#{domain}#{path}")
    end
    
    def build(url, options = {})
      raise NotFound, "No embeddable content at '#{url}'" unless include?(url)
      query = options.merge({:url => url})
      endpoint = @endpoint.clone
      
      if format_in_url?
        format = endpoint["{format}"] = (query[:format] || @format).to_s
        query.delete(:format)
      else
        format = query[:format] ||= @format
      end
      
      query_string = "?" + query.inject("") do |memo, (key, value)|
        "#{key}=#{value}&#{memo}"
      end.chop
      
      URI.parse(endpoint + query_string)      
    end
    
    def raw(url, options = {})
      uri = build(url, options)
      
      res = Net::HTTP.start(uri.host, uri.port) do |http|
        http.get(uri.path + "?" + uri.query)
      end
      
      case res
      when Net::HTTPNotImplemented
        raise UnknownFormat, "The provider doesn't support the '#{format}' format"
      when Net::HTTPNotFound
        raise NotFound, "No embeddable content at '#{url}'"
      else
        res.body
      end
    end
    
    def get(url, options = {})
      OEmbed::Response.new(raw(url, options.merge(:format => :json)), self)
    end                   
    
    def format_in_url?
      @endpoint.include?("{format}")
    end   
    
    def include?(url)
      !!@urls.detect{ |u| u =~ url } 
    end
  end
end<commit_msg>The path needs to be escaped too<commit_after>module OEmbed
  class Provider
    class NotFound<StandardError;end
    class UnknownFormat<StandardError;end
    
    attr_accessor :format, :name, :url, :urls, :endpoint
    
    def initialize(endpoint, format = :json)
      @endpoint = endpoint
      @urls = []
      @format = :json
    end
    
    def <<(url)
      full, scheme, domain, path = *url.match(%r{([^:]*)://?([^/?]*)(.*)})
      domain = Regexp.escape(domain).gsub("\\*", "(.*?)").gsub("(.*?)\\.", "([^\\.]+\\.)?")
      path = Regexp.escape(path).gsub("\\*", "(.*?)")
      @urls << Regexp.new("^#{scheme}://#{domain}#{path}")
    end
    
    def build(url, options = {})
      raise NotFound, "No embeddable content at '#{url}'" unless include?(url)
      query = options.merge({:url => url})
      endpoint = @endpoint.clone
      
      if format_in_url?
        format = endpoint["{format}"] = (query[:format] || @format).to_s
        query.delete(:format)
      else
        format = query[:format] ||= @format
      end
      
      query_string = "?" + query.inject("") do |memo, (key, value)|
        "#{key}=#{value}&#{memo}"
      end.chop
      
      URI.parse(endpoint + query_string)      
    end
    
    def raw(url, options = {})
      uri = build(url, options)
      
      res = Net::HTTP.start(uri.host, uri.port) do |http|
        http.get(uri.path + "?" + uri.query)
      end
      
      case res
      when Net::HTTPNotImplemented
        raise UnknownFormat, "The provider doesn't support the '#{format}' format"
      when Net::HTTPNotFound
        raise NotFound, "No embeddable content at '#{url}'"
      else
        res.body
      end
    end
    
    def get(url, options = {})
      OEmbed::Response.new(raw(url, options.merge(:format => :json)), self)
    end                   
    
    def format_in_url?
      @endpoint.include?("{format}")
    end   
    
    def include?(url)
      !!@urls.detect{ |u| u =~ url } 
    end
  end
end
{'fraction_non_alphanumeric': 0.1760670731707317, 'fraction_numerical': 0.0, 'mean_word_length': 2.1445686900958467, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1839957', 'n_tokens_mistral': 1294, 'n_tokens_neox': 1304, 'n_words': 284}