<commit_before>module MouseHole
class PageHeaders < Array
    def []( k )
        self.assoc(k.to_s.downcase).to_a[1]
    end
    def []=(k, v)
        k = k.to_s.downcase
        if (tmp = self.assoc(k))
            tmp[1] = v
        else
            self << [k, v] 
        end
        v
    end
end
class Page
    attr_accessor :location, :status, :headers, :converter, :document
    def initialize(uri, status, headers)
        @location = URI(uri)
        @status = status
        @headers = PageHeaders[*headers]
        ctype = @headers['Content-Type']
        if ctype
            @converter = Converters.detect_by_mime_type ctype.split(';',2)[0]
        end
    end

    # MrCode's gzip decoding from WonderLand!  Also reads in remainder of the body from the
    # stream.
    def decode(resin)
        body = ''
        resin.read_body do |chunk|
            body += chunk
        end

        case resin['content-encoding']
        when 'gzip':
            gzr = Zlib::GzipReader.new(StringIO.new(body))
            body = gzr.read
            gzr.close<commit_msg> * lib/mouseHole/page.rb: cleanup URLs with unwise characters.<commit_after>module MouseHole
class PageHeaders < Array
    def []( k )
        self.assoc(k.to_s.downcase).to_a[1]
    end
    def []=(k, v)
        k = k.to_s.downcase
        if (tmp = self.assoc(k))
            tmp[1] = v
        else
            self << [k, v] 
        end
        v
    end
end
class Page
    attr_accessor :location, :status, :headers, :converter, :document
    def initialize(uri, status, headers)
        if uri.match(/[#{Regexp::quote('{}|\^[]`')}]/)
            uri = URI.escape(uri)
        end
        @location = URI(uri)
        @status = status
        @headers = PageHeaders[*headers]
        ctype = @headers['Content-Type']
        if ctype
            @converter = Converters.detect_by_mime_type ctype.split(';',2)[0]
        end
    end

    # MrCode's gzip decoding from WonderLand!  Also reads in remainder of the body from the
    # stream.
    def decode(resin)
        body = ''
        resin.read_body do |chunk|
            body += chunk
        end

        case resin['content-encoding']
        when 'gzip':
            gzr = Zlib::GzipReader.new(StringIO.new(body))
            body = gzr.read
            gzr.close
{'fraction_non_alphanumeric': 0.11631944444444445, 'fraction_numerical': 0.003472222222222222, 'mean_word_length': 1.8007290400972054, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5682163', 'n_tokens_mistral': 751, 'n_tokens_neox': 727, 'n_words': 184}