<commit_before>HTTP = require('http')
MIME = require('mime')
URL = require('url')

# If we want to resolve URLs, we need to be able to access window.location?
# Or maybe just have baseURL passed, since we get a new ResourceProxy every
# page load.
# TODO: detect duplicate URLs, fetch them once and cache them.
# TODO: scan @import rules for CSS and fetch subresources.
# TODO: add URLs to ResourceProxy when src attributes are set in advice.
class ResourceProxy
    constructor : (baseURL) ->
        @urls = []
        @baseURL = baseURL
    
    # url - a relative or absolute URL
    addURL : (url) ->
        if /^http/.test(url)
            @urls.push(URL.parse(url))
        else
            @urls.push(URL.parse(URL.resolve(@baseURL, url)))
        return (@urls.length - 1)

    # id - the resource ID to fetch
    # res - the response object to write to.
    fetch : (id, res) ->
        url = @urls[id]
        if !url?
            throw new Error("Tried to fetch invalid id: #{id}")
        type = MIME.lookup(url.href) #TODO: how does this deal with hashes?
        opts =
            host : url.hostname
            port : url.port || 80
            path : url.pathname + (url.search || '')<commit_msg>ResourceProxy: detect duplicate URLs and return existing index.<commit_after>HTTP = require('http')
MIME = require('mime')
URL = require('url')

# TODO: scan @import rules for CSS and fetch subresources.
# TODO: add URLs to ResourceProxy when src attributes are set in advice.
class ResourceProxy
    constructor : (baseURL) ->
        @urlsByIndex = []
        @urlsByName = {}
        @baseURL = baseURL
    
    # url - a relative or absolute URL
    addURL : (url) ->
        parsed = null
        if /^http/.test(url)
            parsed = URL.parse(url)
        else
            parsed = URL.parse(URL.resolve(@baseURL, url))
        href = parsed.href
        return @urlsByName[href] if @urlsByName[href]?
        @urlsByIndex.push(parsed)
        return @urlsByName[href] = @urlsByIndex.length - 1

    # id - the resource ID to fetch
    # res - the response object to write to.
    fetch : (id, res) ->
        url = @urlsByIndex[id]
        if !url?
            throw new Error("Tried to fetch invalid id: #{id}")
        type = MIME.lookup(url.href) #TODO: how does this deal with hashes?
        opts =
            host : url.hostname
            port : url.port || 80
            path : url.pathname + (url.search || '')
{'fraction_non_alphanumeric': 0.11183673469387755, 'fraction_numerical': 0.0024489795918367346, 'mean_word_length': 2.187256176853056, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6418212', 'n_tokens_mistral': 716, 'n_tokens_neox': 725, 'n_words': 253}