<commit_before>require "nokogiri"

module Chip
  class CodeFetcher
    def initialize
      @fetchers = []
      # text/plain
      @fetchers << [/.*/,
                    "text/plain",
                    ->(opts){ opts[:response].body }]
      # text/html <pre>#chip\n puts 'hello, world!'</pre>
      @fetchers << [/.*/,
                    "text/html",
                    ->(opts){ 
                      doc = Nokogiri::HTML::Document.parse(opts[:response].body)
                      doc.xpath('//pre').map do |pre|
                        t = pre.text
                        lines = t.split("\n")
                        if lines.first.include?("chip")
                          return t
                        end
                        return false
                      end
                    }]
    end

    def add(url, content_type, &inspector)
      @fetchers.unshift([url, content_type, inspector])
    end

    def fetch(target_url)
      http_get(target_url) do |response|
        @fetchers.each do |url, content_type, inspector|
          if target_url.match(url) && response.content_type.match(content_type)
            if res = inspector.call(response: response,
                                    url: target_url)
              return res
            end
          end
        end
      end
      raise FetchError, "chip don't have a code fetcher for #{target_url}."
    end

    private
    def http_get(url)
      uri = URI.parse(url)
      http = Net::HTTP.new(uri.host, uri.port)
      http.use_ssl = (uri.scheme == "https")
      http.start do |h|
        res = http.get(uri.path)
        if res.code != "200"
          raise FetchError, "#{url} response code is #{res.code}"
        end
        return yield res
      end
    end
  end
end<commit_msg>for http redirect<commit_after>require "nokogiri"

module Chip
  class CodeFetcher
    def initialize
      @fetchers = []
      # text/plain
      @fetchers << [/.*/,
                    "text/plain",
                    ->(opts){ opts[:response].body }]
      # text/html <pre>#chip\n puts 'hello, world!'</pre>
      @fetchers << [/.*/,
                    "text/html",
                    ->(opts){ 
                      doc = Nokogiri::HTML::Document.parse(opts[:response].body)
                      doc.xpath('//pre').map do |pre|
                        t = pre.text
                        lines = t.split("\n")
                        if lines.first.include?("chip")
                          return t
                        end
                        return false
                      end
                    }]
    end

    def add(url, content_type, &inspector)
      @fetchers.unshift([url, content_type, inspector])
    end

    def fetch(target_url)
      response = http_get(target_url)
      @fetchers.each do |url, content_type, inspector|
        if target_url.match(url) && response.content_type.match(content_type)
          if res = inspector.call(response: response,
                                  url: target_url)
            return res
          end
        end
      end
      raise FetchError, "chip don't have a code fetcher for #{target_url}."
    end

    private
    def http_get(url, limit=10)
      raise FetchError, 'http redirect too deep' if limit == 0
      uri = URI.parse(url)
      http = Net::HTTP.new(uri.host, uri.port)
      http.use_ssl = (uri.scheme == "https")
      res = http.start{|h| h.get(uri.path)}
      case res
      when Net::HTTPSuccess
        return res
      when Net::HTTPRedirection
        return http_get(res['Location'], limit - 1)
      else
        raise FetchError, "#{url} response code is #{res.code}"
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.11348862405200433, 'fraction_numerical': 0.001895991332611051, 'mean_word_length': 1.371868978805395, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6505497', 'n_tokens_mistral': 1079, 'n_tokens_neox': 1026, 'n_words': 249}