<commit_before>var xhr = require('xhr');

var jsonFormatHelper = require('./formatHelpers/json');
var formatHelper = jsonFormatHelper;

var __xhrDebugLevel = 0;
var __xhrPooling = true;
var __xhrPoolSize = 0,
	__xhrPoolFree = [],
	__totalConcurrentXhr = 0,
	__maxConcurrentXhr = 5;

function abortXhr(xhr) {
	if(__xhrDebugLevel >= 1) console.warn('Abort status.', xhr.readyState, xhr.status);
	if(xhr.readyState === 1 || xhr.readyState === 3) {
		if(__xhrDebugLevel >= 2) console.warn('Aborted.', xhr.url);
		xhr.abort();
	} else {
		if(__xhrDebugLevel >= 2) console.warn('Aborted before it started.', xhr.url);
		xhr.onload(new Error('Aborted before it started.'), null, xhr.url);
	}
}

function getXhrLoader(opt, onProgress, onComplete) {

	if (typeof opt === 'string')
		opt = { uri: opt };
	else if (!opt)
		opt = { };

	if (!opt.headers)
		opt.headers = { 'Content-Type': opt.contentType };

	var jsonResponse = /^json$/i.test(opt.responseType);

	if(__xhrPooling && __xhrPoolFree.length > 0) {
		if(__xhrDebugLevel >= 2) console.log('XHR reusing pool for', opt.uri);
		opt.xhr = __xhrPoolFree.shift();
	} else {
		if(__xhrDebugLevel >= 2) console.log('XHR creating new for', opt.uri);
		if(__xhrPooling) __xhrPoolSize++;
	}
	function callbackHandler(err, res, body) {
		if(__xhrPooling) {
			if(__xhrDebugLevel >= 2) console.log('XHR return to pool', _xhr.url);
			__xhrPoolFree.push(_xhr);
		}
		__totalConcurrentXhr--;
		if (err) {
			onComplete(err, null, _xhr.url);
			return;
		}
		if (!/^2/.test(res.statusCode)) {
			onComplete(new Error('http status code: ' + res.statusCode, _xhr.url));<commit_msg>bugfix different responseTypes ruin pooling<commit_after>var xhr = require('xhr');

var jsonFormatHelper = require('./formatHelpers/json');
var formatHelper = jsonFormatHelper;

var __xhrDebugLevel = 0;
var __xhrPooling = true;
var __xhrPoolSize = 0,
	__xhrPoolFree = [],
	__totalConcurrentXhr = 0,
	__maxConcurrentXhr = 5;

function abortXhr(xhr) {
	if(__xhrDebugLevel >= 1) console.warn('Abort status.', xhr.readyState, xhr.status);
	if(xhr.readyState === 1 || xhr.readyState === 3) {
		if(__xhrDebugLevel >= 2) console.warn('Aborted.', xhr.url);
		xhr.abort();
	} else {
		if(__xhrDebugLevel >= 2) console.warn('Aborted before it started.', xhr.url);
		xhr.onload(new Error('Aborted before it started.'), null, xhr.url);
	}
}

function getXhrLoader(opt, onProgress, onComplete) {

	if (typeof opt === 'string')
		opt = { uri: opt };
	else if (!opt)
		opt = { };

	if (!opt.headers)
		opt.headers = { 'Content-Type': opt.contentType };

	var jsonResponse = /^json$/i.test(opt.responseType);

	if(__xhrPooling && __xhrPoolFree.length > 0) {
		if(__xhrDebugLevel >= 2) console.log('XHR reusing pool for', opt.uri);
		opt.xhr = __xhrPoolFree.shift();
		// opt.xhr.responseType = opt.responseType || 'text';
		// cannot change responseType if xhr is Loading or Done...
		// have to rethink pooling when using different responseTypes
	} else {
		if(__xhrDebugLevel >= 2) console.log('XHR creating new for', opt.uri);
		if(__xhrPooling) __xhrPoolSize++;
	}
	function callbackHandler(err, res, body) {
		if(__xhrPooling) {
			if(__xhrDebugLevel >= 2) console.log('XHR return to pool', _xhr.url);
			__xhrPoolFree.push(_xhr);
		}
		__totalConcurrentXhr--;
		if (err) {
			onComplete(err, null, _xhr.url);
			return;
		}
		if (!/^2/.test(res.statusCode)) {
			onComplete(new Error('http status code: ' + res.statusCode, _xhr.url));
{'fraction_non_alphanumeric': 0.16002327611288913, 'fraction_numerical': 0.008146639511201629, 'mean_word_length': 4.355140186915888, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2570718', 'n_tokens_mistral': 1355, 'n_tokens_neox': 1261, 'n_words': 280}