<commit_before>require 'open-uri'

begin
  require 'minigems'
rescue LoadError
  require 'rubygems'
end

require 'memcached'

module Kernel
  private 
  alias openuri_original_open open
  def open(name, *rest, &block)
    if name.respond_to?(:open)
      name.open(*rest, &block)
    elsif name.respond_to?(:to_str) &&
          %r{\A[A-Za-z][A-Za-z0-9+\-\.]*://} =~ name &&
          (uri = URI.parse(name)).respond_to?(:open)
      OpenURI::open(name, *rest, &block)
    else
      open_uri_original_open(name, *rest, &block)
    end
  end
  module_function :open, :openuri_original_open
end

module OpenURI
  alias original_open open #:nodoc:
  def self.open(uri, *rest, &block)
    if Cache.enabled?
      begin
        response = Cache::get(uri.to_s)
      rescue Memcached::NotFound
        response = false
      end
    end
    
    unless
      response = openuri_original_open(uri, *rest).read
      Cache::set(uri.to_s, response) if Cache.enabled?
    end

    response = StringIO.new(response)

    if block_given?
      begin
        yield response
      ensure<commit_msg>I blame my misspent youth.<commit_after>require 'open-uri'

begin
  require 'minigems'
rescue LoadError
  require 'rubygems'
end

require 'memcached'

module Kernel
  private 
  alias openuri_original_open open
  def open(name, *rest, &block)
    if name.respond_to?(:open)
      name.open(*rest, &block)
    elsif name.respond_to?(:to_str) &&
          %r{\A[A-Za-z][A-Za-z0-9+\-\.]*://} =~ name &&
          (uri = URI.parse(name)).respond_to?(:open)
      OpenURI::open(name, *rest, &block)
    else
      open_uri_original_open(name, *rest, &block)
    end
  end
  module_function :open, :openuri_original_open
end

module OpenURI
  alias original_open open #:nodoc:
  def self.open(uri, *rest, &block)
    if Cache.enabled?
      begin
        response = Cache::get(uri.to_s)
      rescue Memcached::NotFound
        response = false
      end
    end
    
    unless response
      response = openuri_original_open(uri, *rest).read
      Cache::set(uri.to_s, response) if Cache.enabled?
    end

    response = StringIO.new(response)

    if block_given?
      begin
        yield response
      ensure
{'fraction_non_alphanumeric': 0.11896080218778486, 'fraction_numerical': 0.0018231540565177757, 'mean_word_length': 2.752136752136752, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2939168', 'n_tokens_mistral': 761, 'n_tokens_neox': 750, 'n_words': 170}