<commit_before>
/**
 * index.js
 *
 * a request API compatible with window.fetch
 */

var parse_url = require('url').parse;
var resolve_url = require('url').resolve;
var http = require('http');
var https = require('https');
var zlib = require('zlib');
var stream = require('stream');

var Response = require('./lib/response');
var Headers = require('./lib/headers');

module.exports = Fetch;

/**
 * Fetch class
 *
 * @param   String   url   Absolute url
 * @param   Object   opts  Fetch options
 * @return  Promise
 */
function Fetch(url, opts) {

	// allow call as function
	if (!(this instanceof Fetch))
		return new Fetch(url, opts);

	// allow custom promise
	if (!Fetch.Promise) {
		throw new Error('native promise missing, set Fetch.Promise to your favorite alternative');
	}

	Response.Promise = Fetch.Promise;

	var self = this;

	// wrap http.request into fetch
	return new Fetch.Promise(function(resolve, reject) {
		var uri = parse_url(url);

		if (!uri.protocol || !uri.hostname) {
			reject(new Error('only absolute urls are supported'));
			return;
		}

		if (uri.protocol !== 'http:' && uri.protocol !== 'https:') {
			reject(new Error('only http(s) protocols are supported'));
			return;
		}

		var request;
		if (uri.protocol === 'https:') {
			request = https.request;
		} else {
			request = http.request;
		}

		opts = opts || {};

		// avoid side-effect on input options
		var options = {
			hostname: uri.hostname
			, port: uri.port
			, path: uri.path
			, auth: uri.auth
			, method: opts.method || 'GET'
			, headers: opts.headers || {}
			, follow: opts.follow !== undefined ? opts.follow : 20
			, counter: opts.counter || 0
			, timeout: opts.timeout || 0
			, compress: opts.compress !== false
			, size: opts.size || 0
			, body: opts.body
			, agent: opts.agent
		};

		// normalize headers
		var headers = new Headers(options.headers);

		if (options.compress) {
			headers.set('accept-encoding', 'gzip,deflate');
		}

		if (!headers.has('user-agent')) {
			headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)');
		}

		if (!headers.has('connection')) {
			headers.set('connection', 'close');
		}

		if (!headers.has('accept')) {
			headers.set('accept', '*/*');
		}

		options.headers = headers.raw();

		// send request
		var req = request(options);

		if (options.timeout) {
			req.once('socket', function(socket) {
				setTimeout(function() {
					req.abort();
					reject(new Error('network timeout at: ' + uri.href));
				}, options.timeout);
			});
		}

		req.on('error', function(err) {
			reject(new Error('request to ' + uri.href + ' failed, reason: ' + err.message));
		});

		req.on('response', function(res) {
			// handle redirect
			if (self.isRedirect(res.statusCode)) {
				if (options.counter >= options.follow) {
					reject(new Error('maximum redirect reached at: ' + uri.href));
					return;
				}
<commit_msg>Clear request timeout after response<commit_after>
/**
 * index.js
 *
 * a request API compatible with window.fetch
 */

var parse_url = require('url').parse;
var resolve_url = require('url').resolve;
var http = require('http');
var https = require('https');
var zlib = require('zlib');
var stream = require('stream');

var Response = require('./lib/response');
var Headers = require('./lib/headers');

module.exports = Fetch;

/**
 * Fetch class
 *
 * @param   String   url   Absolute url
 * @param   Object   opts  Fetch options
 * @return  Promise
 */
function Fetch(url, opts) {

	// allow call as function
	if (!(this instanceof Fetch))
		return new Fetch(url, opts);

	// allow custom promise
	if (!Fetch.Promise) {
		throw new Error('native promise missing, set Fetch.Promise to your favorite alternative');
	}

	Response.Promise = Fetch.Promise;

	var self = this;

	// wrap http.request into fetch
	return new Fetch.Promise(function(resolve, reject) {
		var uri = parse_url(url);

		if (!uri.protocol || !uri.hostname) {
			reject(new Error('only absolute urls are supported'));
			return;
		}

		if (uri.protocol !== 'http:' && uri.protocol !== 'https:') {
			reject(new Error('only http(s) protocols are supported'));
			return;
		}

		var request;
		if (uri.protocol === 'https:') {
			request = https.request;
		} else {
			request = http.request;
		}

		opts = opts || {};

		// avoid side-effect on input options
		var options = {
			hostname: uri.hostname
			, port: uri.port
			, path: uri.path
			, auth: uri.auth
			, method: opts.method || 'GET'
			, headers: opts.headers || {}
			, follow: opts.follow !== undefined ? opts.follow : 20
			, counter: opts.counter || 0
			, timeout: opts.timeout || 0
			, compress: opts.compress !== false
			, size: opts.size || 0
			, body: opts.body
			, agent: opts.agent
		};

		// normalize headers
		var headers = new Headers(options.headers);

		if (options.compress) {
			headers.set('accept-encoding', 'gzip,deflate');
		}

		if (!headers.has('user-agent')) {
			headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)');
		}

		if (!headers.has('connection')) {
			headers.set('connection', 'close');
		}

		if (!headers.has('accept')) {
			headers.set('accept', '*/*');
		}

		options.headers = headers.raw();

		// send request
		var req = request(options);
		var reqTimeout;

		if (options.timeout) {
			req.once('socket', function(socket) {
				reqTimeout = setTimeout(function() {
					req.abort();
					reject(new Error('network timeout at: ' + uri.href));
				}, options.timeout);
			});
		}

		req.on('error', function(err) {
			clearTimeout(reqTimeout);
			reject(new Error('request to ' + uri.href + ' failed, reason: ' + err.message));
		});

		req.on('response', function(res) {
			clearTimeout(reqTimeout);

			// handle redirect
			if (self.isRedirect(res.statusCode)) {
				if (options.counter >= options.follow) {
					reject(new Error('maximum redirect reached at: ' + uri.href));
					return;
				}

{'fraction_non_alphanumeric': 0.16669490089784855, 'fraction_numerical': 0.002371675419278333, 'mean_word_length': 3.2474820143884893, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5943724', 'n_tokens_mistral': 2153, 'n_tokens_neox': 1859, 'n_words': 481}