<commit_before>
// Load modules

var Events = require('events');
var Url = require('url');
var Http = require('http');
var Https = require('https');
var Stream = require('stream');
var Hoek = require('hoek');
var Boom = require('boom');
var Payload = require('./payload');
var Recorder = require('./recorder');
var Tap = require('./tap');


// Declare internals

var internals = {
    jsonRegex: /^application\/[a-z.+-]*json$/,
    shallowOptions: ['agent', 'payload', 'downstreamRes', 'redirected']
};


// new instance is exported as module.exports

internals.Client = function (defaults) {

    Events.EventEmitter.call(this);

    this.agents = {
        https: new Https.Agent({ maxSockets: Infinity }),
        http: new Http.Agent({ maxSockets: Infinity }),
        httpsAllowUnauthorized: new Https.Agent({ maxSockets: Infinity, rejectUnauthorized: false })
    };

    this._defaults = defaults || {};
};

Hoek.inherits(internals.Client, Events.EventEmitter);


internals.Client.prototype.defaults = function (options) {

    options = Hoek.applyToDefaultsWithShallow(options, this._defaults, internals.shallowOptions);
    return new internals.Client(options);
};


internals.resolveUrl = function (baseUrl, path) {

    if (!path) {
        return baseUrl;
    }

    var parsedBase = Url.parse(baseUrl);
    var parsedPath = Url.parse(path);

    parsedBase.pathname += parsedPath.pathname;
    parsedBase.pathname = parsedBase.pathname.replace(/[/]{2,}/g, '/');
    parsedBase.search = parsedPath.search;      // Always use the querystring from the path argument

    return Url.format(parsedBase);
};


internals.Client.prototype.request = function (method, url, options, callback, _trace) {

    var self = this;

    options = Hoek.applyToDefaultsWithShallow(options || {}, this._defaults, internals.shallowOptions);

    Hoek.assert(options.payload === null || options.payload === undefined || typeof options.payload === 'string' ||
        options.payload instanceof Stream || Buffer.isBuffer(options.payload),
        'options.payload must be a string, a Buffer, or a Stream');

    Hoek.assert((options.agent === undefined || options.agent === null) || (typeof options.rejectUnauthorized !== 'boolean'),
        'options.agent cannot be set to an Agent at the same time as options.rejectUnauthorized is set');

    Hoek.assert(options.redirected === undefined || options.redirected === null || typeof options.redirected === 'function',
        'options.redirected must be a function');

    if (options.baseUrl) {
        url = internals.resolveUrl(options.baseUrl, url);
        delete options.baseUrl;
    }

    var uri = Url.parse(url);
    var timeoutId;
    uri.method = method.toUpperCase();
    uri.headers = options.headers;

    var payloadSupported = (uri.method !== 'GET' && uri.method !== 'HEAD' && options.payload !== null && options.payload !== undefined);
    if (payloadSupported &&
        (typeof options.payload === 'string' || Buffer.isBuffer(options.payload))) {

        uri.headers = Hoek.clone(uri.headers) || {};
        uri.headers['Content-Length'] = Buffer.isBuffer(options.payload) ? options.payload.length : Buffer.byteLength(options.payload);
    }

    var redirects = (options.hasOwnProperty('redirects') ? options.redirects : false);      // Needed to allow 0 as valid value when passed recursively

    _trace = (_trace || []);
    _trace.push({ method: uri.method, url: url });

    var client = (uri.protocol === 'https:' ? Https : Http);

    if (options.rejectUnauthorized !== undefined && uri.protocol === 'https:') {
        uri.agent = options.rejectUnauthorized ? this.agents.https : this.agents.httpsAllowUnauthorized;
    }
    else if (options.agent || options.agent === false) {
        uri.agent = options.agent;
    }
    else {
        uri.agent = uri.protocol === 'https:' ? this.agents.https : this.agents.http;
    }

    if (options.secureProtocol !== undefined) {
        uri.secureProtocol = options.secureProtocol;
    }

    var start = Date.now();
    var req = client.request(uri);

    var shadow = null;                                                                      // A copy of the streamed request payload when redirects are enabled

    // Register handlers

    var finish = function (err, res) {

        if (!callback || err) {
            req.abort();
        }

        req.removeListener('response', onResponse);
        req.removeListener('error', onError);
        req.on('error', Hoek.ignore);
        clearTimeout(timeoutId);
        self.emit('response', err, req, res, start, uri);

        if (callback) {
            return callback(err, res);
        }
    };

    finish = Hoek.once(finish);

    var onError = function (err) {

        err.trace = _trace;
        return finish(Boom.badGateway('Client request error', err));
    };

    req.once('error', onError);

    var onResponse = function (res) {

        // Pass-through response

        var statusCode = res.statusCode;

        if (redirects === false ||
            [301, 302, 307, 308].indexOf(statusCode) === -1) {

            return finish(null, res);
        }

        // Redirection

        var redirectMethod = (statusCode === 301 || statusCode === 302 ? 'GET' : uri.method);
        var location = res.headers.location;

        res.destroy();

        if (redirects === 0) {
            return finish(Boom.badGateway('Maximum redirections reached', _trace));
        }

        if (!location) {
            return finish(Boom.badGateway('Received redirection without location', _trace));
        }

        if (!/^https?:/i.test(location)) {
            location = Url.resolve(uri.href, location);
        }

        var redirectOptions = Hoek.cloneWithShallow(options, internals.shallowOptions);

        redirectOptions.payload = shadow || options.payload;         // shadow must be ready at this point if set
        redirectOptions.redirects = --redirects;

        var redirectReq = self.request(redirectMethod, location, redirectOptions, finish, _trace);

        if (options.redirected) {
            options.redirected(statusCode, location, redirectReq);
        }
    };

    req.once('response', onResponse);

    if (options.timeout) {
        timeoutId = setTimeout(function () {

            return finish(Boom.gatewayTimeout('Client request timeout'));
        }, options.timeout);
        delete options.timeout;
    }

    // Write payload

    if (payloadSupported) {
        if (options.payload instanceof Stream) {
            var stream = options.payload;<commit_msg>Add beforeRedirect option<commit_after>
// Load modules

var Events = require('events');
var Url = require('url');
var Http = require('http');
var Https = require('https');
var Stream = require('stream');
var Hoek = require('hoek');
var Boom = require('boom');
var Payload = require('./payload');
var Recorder = require('./recorder');
var Tap = require('./tap');


// Declare internals

var internals = {
    jsonRegex: /^application\/[a-z.+-]*json$/,
    shallowOptions: ['agent', 'payload', 'downstreamRes', 'beforeRedirect', 'redirected']
};


// new instance is exported as module.exports

internals.Client = function (defaults) {

    Events.EventEmitter.call(this);

    this.agents = {
        https: new Https.Agent({ maxSockets: Infinity }),
        http: new Http.Agent({ maxSockets: Infinity }),
        httpsAllowUnauthorized: new Https.Agent({ maxSockets: Infinity, rejectUnauthorized: false })
    };

    this._defaults = defaults || {};
};

Hoek.inherits(internals.Client, Events.EventEmitter);


internals.Client.prototype.defaults = function (options) {

    options = Hoek.applyToDefaultsWithShallow(options, this._defaults, internals.shallowOptions);
    return new internals.Client(options);
};


internals.resolveUrl = function (baseUrl, path) {

    if (!path) {
        return baseUrl;
    }

    var parsedBase = Url.parse(baseUrl);
    var parsedPath = Url.parse(path);

    parsedBase.pathname += parsedPath.pathname;
    parsedBase.pathname = parsedBase.pathname.replace(/[/]{2,}/g, '/');
    parsedBase.search = parsedPath.search;      // Always use the querystring from the path argument

    return Url.format(parsedBase);
};


internals.Client.prototype.request = function (method, url, options, callback, _trace) {

    var self = this;

    options = Hoek.applyToDefaultsWithShallow(options || {}, this._defaults, internals.shallowOptions);

    Hoek.assert(options.payload === null || options.payload === undefined || typeof options.payload === 'string' ||
        options.payload instanceof Stream || Buffer.isBuffer(options.payload),
        'options.payload must be a string, a Buffer, or a Stream');

    Hoek.assert((options.agent === undefined || options.agent === null) || (typeof options.rejectUnauthorized !== 'boolean'),
        'options.agent cannot be set to an Agent at the same time as options.rejectUnauthorized is set');

    Hoek.assert(options.beforeRedirect === undefined || options.beforeRedirect === null || typeof options.beforeRedirect === 'function',
        'options.beforeRedirect must be a function');
    
    Hoek.assert(options.redirected === undefined || options.redirected === null || typeof options.redirected === 'function',
        'options.redirected must be a function');

    if (options.baseUrl) {
        url = internals.resolveUrl(options.baseUrl, url);
        delete options.baseUrl;
    }

    var uri = Url.parse(url);
    var timeoutId;
    uri.method = method.toUpperCase();
    uri.headers = options.headers;

    var payloadSupported = (uri.method !== 'GET' && uri.method !== 'HEAD' && options.payload !== null && options.payload !== undefined);
    if (payloadSupported &&
        (typeof options.payload === 'string' || Buffer.isBuffer(options.payload))) {

        uri.headers = Hoek.clone(uri.headers) || {};
        uri.headers['Content-Length'] = Buffer.isBuffer(options.payload) ? options.payload.length : Buffer.byteLength(options.payload);
    }

    var redirects = (options.hasOwnProperty('redirects') ? options.redirects : false);      // Needed to allow 0 as valid value when passed recursively

    _trace = (_trace || []);
    _trace.push({ method: uri.method, url: url });

    var client = (uri.protocol === 'https:' ? Https : Http);

    if (options.rejectUnauthorized !== undefined && uri.protocol === 'https:') {
        uri.agent = options.rejectUnauthorized ? this.agents.https : this.agents.httpsAllowUnauthorized;
    }
    else if (options.agent || options.agent === false) {
        uri.agent = options.agent;
    }
    else {
        uri.agent = uri.protocol === 'https:' ? this.agents.https : this.agents.http;
    }

    if (options.secureProtocol !== undefined) {
        uri.secureProtocol = options.secureProtocol;
    }

    var start = Date.now();
    var req = client.request(uri);

    var shadow = null;                                                                      // A copy of the streamed request payload when redirects are enabled

    // Register handlers

    var finish = function (err, res) {

        if (!callback || err) {
            req.abort();
        }

        req.removeListener('response', onResponse);
        req.removeListener('error', onError);
        req.on('error', Hoek.ignore);
        clearTimeout(timeoutId);
        self.emit('response', err, req, res, start, uri);

        if (callback) {
            return callback(err, res);
        }
    };

    finish = Hoek.once(finish);

    var onError = function (err) {

        err.trace = _trace;
        return finish(Boom.badGateway('Client request error', err));
    };

    req.once('error', onError);

    var onResponse = function (res) {

        // Pass-through response

        var statusCode = res.statusCode;

        if (redirects === false ||
            [301, 302, 307, 308].indexOf(statusCode) === -1) {

            return finish(null, res);
        }

        // Redirection

        var redirectMethod = (statusCode === 301 || statusCode === 302 ? 'GET' : uri.method);
        var location = res.headers.location;

        res.destroy();

        if (redirects === 0) {
            return finish(Boom.badGateway('Maximum redirections reached', _trace));
        }

        if (!location) {
            return finish(Boom.badGateway('Received redirection without location', _trace));
        }

        if (!/^https?:/i.test(location)) {
            location = Url.resolve(uri.href, location);
        }

        var redirectOptions = Hoek.cloneWithShallow(options, internals.shallowOptions);

        redirectOptions.payload = shadow || options.payload;         // shadow must be ready at this point if set
        redirectOptions.redirects = --redirects;
        
        if (options.beforeRedirect) {
            options.beforeRedirect(redirectMethod, statusCode, location, redirectOptions);
        }

        var redirectReq = self.request(redirectMethod, location, redirectOptions, finish, _trace);

        if (options.redirected) {
            options.redirected(statusCode, location, redirectReq);
        }
    };

    req.once('response', onResponse);

    if (options.timeout) {
        timeoutId = setTimeout(function () {

            return finish(Boom.gatewayTimeout('Client request timeout'));
        }, options.timeout);
        delete options.timeout;
    }

    // Write payload

    if (payloadSupported) {
        if (options.payload instanceof Stream) {
            var stream = options.payload;
{'fraction_non_alphanumeric': 0.13608110108036112, 'fraction_numerical': 0.003255882788219624, 'mean_word_length': 2.754166666666667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6132063', 'n_tokens_mistral': 3809, 'n_tokens_neox': 3655, 'n_words': 922}