<commit_before>var extend = require('util')._extend;
var util = require('../../util');
var agentConfig = {maxSockets: 8};
var httpAgent = new (require('http').Agent)(agentConfig);
var httpsAgent = new (require('https').Agent)(agentConfig);
var config = util.config;

module.exports = function(req, res, next) {
	var protocol = req.options && req.options.protocol;
	if (!util.isWebProtocol(protocol)) {
		next();
		return;
	}
	
	var options = extend({}, req.options);
	var headers = extend({}, req.headers);
	
	headers.host = headers.Host = options.hostname ? options.hostname + (options.port ? ':' + options.port : '') 
			: headers.host;//tencent的服务器不识别小写的host
	if (req.proxy) {
		headers['x-from-proxy'] = config.name;
	}
	options.agent = protocol == 'https:' ? httpsAgent : httpAgent;
	options.rejectUnauthorized = false;
	options.headers = headers;
	options.method = req.method;
	options.hostname = null;
	req.request(options);
};<commit_msg>proxy<commit_after>var extend = require('util')._extend;
var util = require('../../util');
var agentConfig = {maxSockets: 8};
var httpAgent = new (require('http').Agent)(agentConfig);
var httpsAgent = new (require('https').Agent)(agentConfig);
var config = util.config;

module.exports = function(req, res, next) {
	var protocol = req.options && req.options.protocol;
	if (!util.isWebProtocol(protocol)) {
		next();
		return;
	}
	
	var options = extend({}, req.options);
	var headers = extend({}, req.headers);
	
	headers.host = headers.Host = options.hostname ? options.hostname + (options.port ? ':' + options.port : '') 
			: headers.host;//tencent的服务器不识别小写的host
	if (options.proxy) {
		headers['x-from-proxy'] = config.name;
	}
	options.agent = protocol == 'https:' ? httpsAgent : httpAgent;
	options.rejectUnauthorized = false;
	options.headers = headers;
	options.method = req.method;
	options.hostname = null;
	req.request(options);
};
{'fraction_non_alphanumeric': 0.1716101694915254, 'fraction_numerical': 0.001059322033898305, 'mean_word_length': 4.638805970149254, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4481385', 'n_tokens_mistral': 625, 'n_tokens_neox': 614, 'n_words': 132}