<commit_before>var url = require('url'),
    http = require('http'),
    https = require('https'),
    zlib = require('zlib'),

    _ = require('underscore'),

    cookies = require('./cookies'),
    encoding = require('./encodingstream'),
    urlPrefix = require('./urlprefixstream'),
    metaRobots = require('./metarobotsstream'),
    googleAnalytics = require('./googleanalyticsstream');

var config = {
    prefix: '/proxy/'
};

function setConfig(conf) {
    _.defaults(conf, config);
    setupUrlPrefix();
}

function setupUrlPrefix() {
    urlPrefix.setDefaults({
        prefix: config.prefix
    });
}
setupUrlPrefix();


var portmap = {
    "http:": 80,
    "https:": 443
};


/**
 * Makes the outgoing request and relays it to the client, modifying it along the way if necessary
 */
function proxy(uri, request, response) {
    request.session = request.session || {};

    // redirect urls like /proxy/http://asdf.com to /proxy/http://asdf.com/ to make relative image paths work
    if (uri.pathname == "/" && request.url.substr(-1) != "/") {
        return response.redirectTo(request.url + "/");
    }

    uri.port = uri.port || portmap[uri.protocol];
    uri.pathname = uri.search ? uri.pathname + uri.search : uri.pathname;

    var headers = copy(request.headers);

    delete headers.host;

    // todo: grab any new cookies in headers.cookie (set by JS) and store them in the session
    // (assume / path and same domain as request's referer)
    headers.cookie = cookies.get(request, uri);

    //console.log("sending these cookies: " + headers.cookie);

    // overwrite the referer with the correct referer
    if (request.headers.referer) {
        headers.referer = getRealUrl(request.headers.referer);
    }<commit_msg>Fix edge cases with url formatting when a not-quite-complete url is entered into the form. Fixes #27<commit_after>var url = require('url'),
    http = require('http'),
    https = require('https'),
    zlib = require('zlib'),

    _ = require('underscore'),

    cookies = require('./cookies'),
    encoding = require('./encodingstream'),
    urlPrefix = require('./urlprefixstream'),
    metaRobots = require('./metarobotsstream'),
    googleAnalytics = require('./googleanalyticsstream');

var config = {
    prefix: '/proxy/'
};

function setConfig(conf) {
    _.defaults(conf, config);
    setupUrlPrefix();
}

function setupUrlPrefix() {
    urlPrefix.setDefaults({
        prefix: config.prefix
    });
}
setupUrlPrefix();


var portmap = {
    "http:": 80,
    "https:": 443
};


/**
 * Makes the outgoing request and relays it to the client, modifying it along the way if necessary
 */
function proxy(uri, request, response) {
    request.session = request.session || {};

    // redirect urls like /proxy/http://asdf.com to /proxy/http://asdf.com/ to make relative image paths work
    var formatted = url.format(uri);
    if (formatted != request.url.substr(config.prefix.length)) {
        return response.redirectTo(formatted);
    }

    uri.port = uri.port || portmap[uri.protocol];
    uri.pathname = uri.search ? uri.pathname + uri.search : uri.pathname;

    var headers = copy(request.headers);

    delete headers.host;

    // todo: grab any new cookies in headers.cookie (set by JS) and store them in the session
    // (assume / path and same domain as request's referer)
    headers.cookie = cookies.get(request, uri);

    //console.log("sending these cookies: " + headers.cookie);

    // overwrite the referer with the correct referer
    if (request.headers.referer) {
        headers.referer = getRealUrl(request.headers.referer);
    }
{'fraction_non_alphanumeric': 0.14178897812240376, 'fraction_numerical': 0.003600110772639158, 'mean_word_length': 3.1374570446735395, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2071184', 'n_tokens_mistral': 1086, 'n_tokens_neox': 1042, 'n_words': 291}