<commit_before>"use strict";
var callHandler    = require("../internal/callHandler");
var getHtmlFromUrl = require("../internal/getHtmlFromUrl");
var parseOptions   = require("../internal/parseOptions");

var HtmlChecker = require("./HtmlChecker");

var RequestQueue = require("limited-request-queue");



function HtmlUrlChecker(options, handlers)
{
	var thisObj = this;
	
	// Not necessary, but creates the variables
	reset(this);
	
	this.handlers = handlers || {};
	this.options = parseOptions(options);
	
	this.htmlUrlQueue = new RequestQueue(
	{
		maxSockets: 1,
		maxSocketsPerHost: Infinity,
		rateLimit: this.options.rateLimit
	},
	{
		item: function(input, done)
		{
			thisObj.currentCustomData = input.data.customData;
			thisObj.currentDone = done;
			thisObj.currentHtmlUrl = input.url;
			
			// TODO :: make callback reusable
			getHtmlFromUrl(thisObj.currentHtmlUrl, thisObj.options, function(error, htmlString, responseUrl)
			{
				if (error !== null)
				{
					complete(thisObj, error);
					return;
				}
				
				thisObj.htmlChecker.scan(htmlString, responseUrl);
				
				// Internal event (undocumented)
				callHandler.sync(thisObj.handlers.html, [input.id]);
			});
		},
		end: function()
		{
			// Clear references for garbage collection
			reset(thisObj);
			
			callHandler.sync(thisObj.handlers.end);
		}
	});
	
	this.htmlChecker = new HtmlChecker(this.options,
	{
		link: function(result)
		{
			callHandler.sync(thisObj.handlers.link, [result, thisObj.currentCustomData]);
		},
		complete: function()
		{
			complete(thisObj, null);
		}
	});
}



HtmlChecker.prototype.clearCache = function()
{
	this.urlChecker.clearCache();
};



HtmlUrlChecker.prototype.dequeue = function(id)
{
	return this.htmlUrlQueue.dequeue(id);
};



HtmlUrlChecker.prototype.enqueue = function(htmlUrl, customData)
{
	return this.htmlUrlQueue.enqueue(
	{
		url: htmlUrl,
		data: { customData:customData }
	});
};



HtmlUrlChecker.prototype.length = function()
{
	return this.htmlUrlQueue.length();
};



HtmlUrlChecker.prototype.numActiveItems = function()
{
	return this.htmlUrlQueue.numActive();
};



HtmlUrlChecker.prototype.numActiveLinks = function()
{
	return this.htmlChecker.numActive();
};



HtmlUrlChecker.prototype.pause = function()
{
	this.htmlChecker.pause();
	this.htmlUrlQueue.pause();
};



HtmlUrlChecker.prototype.resume = function()
{
	this.htmlChecker.resume();
	this.htmlUrlQueue.resume();
};



//::: PRIVATE FUNCTIONS



function complete(instance, error)
{
	callHandler.sync(instance.handlers.item, [error, instance.currentHtmlUrl, instance.currentCustomData]);
	
	// Auto-starts next queue item, if any
	// If not, fires "end"
	instance.currentDone();
}



function reset(instance)
{
	instance.currentCustomData = null;
	instance.currentDone = null;
	instance.currentHtmlUrl = null;
}



module.exports = HtmlUrlChecker;
<commit_msg>Fixes incorrectly named public function<commit_after>"use strict";
var callHandler    = require("../internal/callHandler");
var getHtmlFromUrl = require("../internal/getHtmlFromUrl");
var parseOptions   = require("../internal/parseOptions");

var HtmlChecker = require("./HtmlChecker");

var RequestQueue = require("limited-request-queue");



function HtmlUrlChecker(options, handlers)
{
	var thisObj = this;
	
	// Not necessary, but creates the variables
	reset(this);
	
	this.handlers = handlers || {};
	this.options = parseOptions(options);
	
	this.htmlUrlQueue = new RequestQueue(
	{
		maxSockets: 1,
		maxSocketsPerHost: Infinity,
		rateLimit: this.options.rateLimit
	},
	{
		item: function(input, done)
		{
			thisObj.currentCustomData = input.data.customData;
			thisObj.currentDone = done;
			thisObj.currentHtmlUrl = input.url;
			
			// TODO :: make callback reusable
			getHtmlFromUrl(thisObj.currentHtmlUrl, thisObj.options, function(error, htmlString, responseUrl)
			{
				if (error !== null)
				{
					complete(thisObj, error);
					return;
				}
				
				thisObj.htmlChecker.scan(htmlString, responseUrl);
				
				// Internal event (undocumented)
				callHandler.sync(thisObj.handlers.html, [input.id]);
			});
		},
		end: function()
		{
			// Clear references for garbage collection
			reset(thisObj);
			
			callHandler.sync(thisObj.handlers.end);
		}
	});
	
	this.htmlChecker = new HtmlChecker(this.options,
	{
		link: function(result)
		{
			callHandler.sync(thisObj.handlers.link, [result, thisObj.currentCustomData]);
		},
		complete: function()
		{
			complete(thisObj, null);
		}
	});
}



HtmlUrlChecker.prototype.clearCache = function()
{
	this.htmlChecker.clearCache();
};



HtmlUrlChecker.prototype.dequeue = function(id)
{
	return this.htmlUrlQueue.dequeue(id);
};



HtmlUrlChecker.prototype.enqueue = function(htmlUrl, customData)
{
	return this.htmlUrlQueue.enqueue(
	{
		url: htmlUrl,
		data: { customData:customData }
	});
};



HtmlUrlChecker.prototype.length = function()
{
	return this.htmlUrlQueue.length();
};



HtmlUrlChecker.prototype.numActiveItems = function()
{
	return this.htmlUrlQueue.numActive();
};



HtmlUrlChecker.prototype.numActiveLinks = function()
{
	return this.htmlChecker.numActive();
};



HtmlUrlChecker.prototype.pause = function()
{
	this.htmlChecker.pause();
	this.htmlUrlQueue.pause();
};



HtmlUrlChecker.prototype.resume = function()
{
	this.htmlChecker.resume();
	this.htmlUrlQueue.resume();
};



//::: PRIVATE FUNCTIONS



function complete(instance, error)
{
	callHandler.sync(instance.handlers.item, [error, instance.currentHtmlUrl, instance.currentCustomData]);
	
	// Auto-starts next queue item, if any
	// If not, fires "end"
	instance.currentDone();
}



function reset(instance)
{
	instance.currentCustomData = null;
	instance.currentDone = null;
	instance.currentHtmlUrl = null;
}



module.exports = HtmlUrlChecker;

{'fraction_non_alphanumeric': 0.13444213444213443, 'fraction_numerical': 0.0003465003465003465, 'mean_word_length': 4.168307967770815, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1754973', 'n_tokens_mistral': 2064, 'n_tokens_neox': 1860, 'n_words': 326}