<commit_before>#! /usr/bin/env ruby
# encoding: UTF-8

require 'net/http'
require 'uri'
require 'stringio'
require 'zlib'
require 'rss'
require 'yaml'

def http_get_chunked (url, prefix = nil, headers = {}, &block)
    url = URI.parse url
    STDERR.print "#{prefix} " if prefix

    chunk_loop = %w{/ - \\ |}
    chunk_i = 0

    body = ""
    Net::HTTP.start(url.host, url.port) do |http|
        http.request_get(url.request_uri, headers) do |res|
            unless res.is_a? Net::HTTPSuccess
                raise Error.new("Failed to get #{url} (HTTP #{res})")
            else
                len = res['content-length'] ? res['content-length'].to_i : nil
                res.read_body do |chunk|
                    body += chunk
                    if len
                        STDERR.printf "\r%s %.2f%%", prefix, body.size * 100.0 / len if prefix
                    else
                        STDERR.print "\r#{prefix} #{chunk_loop[chunk_i]}"
                        chunk_i = (chunk_i + 1) % chunk_loop.length
                    end
                    yield chunk, res if block
                end
                # extra whitespace to clean the 'junk' from the progress display
                STDERR.puts "\r#{prefix} 100%   \r#{prefix} 100%" if prefix
            end
        end
    end
    body
end

def http_get (url, prefix = nil, &block)
    gzip = nil
    body = http_get_chunked(url, prefix, { 'Accept-Encoding' => "gzip" }) do |x,res|
        gzip = res["Content-Encoding"] && res["Content-Encoding"] == "gzip" if gzip.nil?
    end
    Zlib::GzipReader.wrap(StringIO.new body.dup) {|gz| body = gz.read} if gzip

    if block
        yield body
    else
        body
    end
end<commit_msg>http_get_chunked: don't print spinner if no prefix was given<commit_after>#! /usr/bin/env ruby
# encoding: UTF-8

require 'net/http'
require 'uri'
require 'stringio'
require 'zlib'
require 'rss'
require 'yaml'

def http_get_chunked (url, prefix = nil, headers = {}, &block)
    url = URI.parse url
    STDERR.print "#{prefix} " if prefix

    chunk_loop = %w{/ - \\ |}
    chunk_i = 0

    body = ""
    Net::HTTP.start(url.host, url.port) do |http|
        http.request_get(url.request_uri, headers) do |res|
            unless res.is_a? Net::HTTPSuccess
                raise Error.new("Failed to get #{url} (HTTP #{res})")
            else
                len = res['content-length'] ? res['content-length'].to_i : nil
                res.read_body do |chunk|
                    body += chunk
                    if len
                        STDERR.printf "\r%s %.2f%%", prefix, body.size * 100.0 / len if prefix
                    elsif prefix
                        STDERR.print "\r#{prefix} #{chunk_loop[chunk_i]}"
                        chunk_i = (chunk_i + 1) % chunk_loop.length
                    end
                    yield chunk, res if block
                end
                # extra whitespace to clean the 'junk' from the progress display
                STDERR.puts "\r#{prefix} 100%   \r#{prefix} 100%" if prefix
            end
        end
    end
    body
end

def http_get (url, prefix = nil, &block)
    gzip = nil
    body = http_get_chunked(url, prefix, { 'Accept-Encoding' => "gzip" }) do |x,res|
        gzip = res["Content-Encoding"] && res["Content-Encoding"] == "gzip" if gzip.nil?
    end
    Zlib::GzipReader.wrap(StringIO.new body.dup) {|gz| body = gz.read} if gzip

    if block
        yield body
    else
        body
    end
end
{'fraction_non_alphanumeric': 0.12310334955625536, 'fraction_numerical': 0.008016032064128256, 'mean_word_length': 1.7296875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6448622', 'n_tokens_mistral': 1138, 'n_tokens_neox': 1076, 'n_words': 306}