<commit_before>// ---------------------------------------

if (require.main === module)
	throw (new Error ());

// ---------------------------------------

var http = require ("http");
var url = require ("url");

var configuration = require ("./configuration");
var store = require ("./store-lib")
var transcript = require ("./transcript") (module, configuration.libTranscriptLevel);

// ---------------------------------------

function _fetch (_url, _context, _callback) {
	var _task = {};
	_task.url = _url;
	_task.context = _context;
	_task.callback = _callback;
	_task.type = configuration.fetchTaskType;
	_task.feedKey = store.generateFeedKey (_task.url);
	_task.taskKey = store.generateFeedTaskKey (_task.url, _task.type);
	_task.contentType = "application/atom+xml; charset=utf-8";
	_doFetchStep1 (_task);
}

function _doFetchStep1 (_task) {
	transcript.traceDebugging ("fetching `%s` step 1 (fetching latest fetch task)...", _task.url);
	store.fetchFeedTask (_task.context.riak, _task.taskKey,
			function (_error, _previousTaskOutcome, _riakMetaData) {
				if (_error !== null) {
					_task.error = _error;
					_onFetchError (_task);
				} else {
					_task.taskRiakMetaData = _riakMetaData;
					_task.previousTaskOutcome = _previousTaskOutcome;
					if ((_task.previousTaskOutcome !== null)
							&& (
								(_task.previousTaskOutcome.currentEtag === undefined)
								|| (_task.previousTaskOutcome.currentTimestamp === undefined)
								|| (_task.previousTaskOutcome.currentData === undefined)
								|| (_task.previousTaskOutcome.updatedTimestamp === undefined)
								|| (_task.previousTaskOutcome.outcome === undefined)
								|| (_task.previousTaskOutcome.error === undefined)))
						_task.previousTaskOutcome = null;
					if ((_task.previousTaskOutcome === null)
							|| ((new Date () .getTime () - _task.previousTaskOutcome.currentTimestamp) > configuration.fetcherMinFetchAge))
						_doFetchStep2 (_task);
					else {
						_task.error = {reason : "rejected", cause : "fetchAge <="};
						_onFetchError (_task);
					}
				}
			});
}

function _doFetchStep2 (_task) {
	transcript.traceDebugging ("fetching `%s` step 2 (fetching latest data)...", _task.url);
	_fetchUrl (_task.url, _task.contentType,
			_task.previousTaskOutcome ? _task.previousTaskOutcome.previousEtag : null,
			_task.previousTaskOutcome ? _task.previousTaskOutcome.previousTimestamp : null,
			function (_error, _outcome, _data) {
				if (_error !== null) {
					_task.error = _error;
					_task.dataOutcome = null;
					_task.data = null;
					_onFetchStep3b (_task);
				} else {
					_task.dataOutcome = _outcome;
					_task.data = _data;
					if (_task.data !== null)
						_onFetchStep3a (_task);
					else
						_onFetchStep3b (_task);
				}
			});
}

function _onFetchStep3a (_task) {
	transcript.traceDebugging ("fetching `%s` step 3a (updating data)...", _task.url);
	_task.dataKey = store.generateFeedDataKey (_task.url, _task.data);
	store.updateFeedData (_task.context.riak, _task.dataKey, _task.data,
			{
					contentType : _task.dataOutcome.contentType,
					usermeta : {feedKey : _task.feedKey, feedTaskKey : _task.taskKey, feedTaskType : _task.type}},
			function (_error, _riakMetaData) {
				if (_error !== null) {
					_task.error = _error;
					_onFetchError (_task);
				} else {
					_task.dataRiakMetaData = _riakMetaData;
					_onFetchStep4 (_task);
				}
			});
}

function _onFetchStep3b (_task) {
	_task.dataKey = _task.previousTaskOutcome ? _task.previousTaskOutcome.currentData : null;
	_task.dataRiakMetaData = null;
	_onFetchStep4 (_task);
}

function _onFetchStep4 (_task) {
	transcript.traceDebugging ("fetching `%s` step 4 (updating fetch task)...", _task.url);
	_task.currentTaskOutcome = {
		key : _task.taskKey,
		type : _task.type,
		feed : _task.feedKey,
		url : _task.url,
		currentData : _task.dataKey,
		currentEtag : _task.dataOutcome ? _task.dataOutcome.currentEtag : null,
		currentTimestamp : new Date () .getTime (),
		previousData : _task.previousTaskOutcome ? _task.previousTaskOutcome.currentData : null,
		previousEtag : _task.previousTaskOutcome ? _task.previousTaskOutcome.currentEtag : null,
		previousTimestamp : _task.previousTaskOutcome ? _task.previousTaskOutcome.currentTimestamp : null,
		updatedTimestamp : _task.previousTaskOutcome ? _task.previousTaskOutcome.updatedTimestamp : null,
		outcome : _task.dataOutcome,
		error : _task.error ? _task.error : null
	};
	if (_task.currentTaskOutcome.currentData != _task.currentTaskOutcome.previousData)
		_task.currentTaskOutcome.updatedTimestamp = _task.currentTaskOutcome.currentTimestamp;
	_task.taskRiakMetaData.contentType = "application/json";
	store.updateFeedTask (_task.context.riak, _task.taskKey, _task.currentTaskOutcome, _task.taskRiakMetaData,
			function (_error, _riakMetaData) {
				if (_error !== null) {
					_task.error = _error;
					_onFetchError (_task);
				} else {
					_task.taskRiakMetaData = _riakMetaData;
					transcript.traceDebugging ("succeeded fetching `%s`", _task.url);
					_task.callback (null, _task.currentTaskOutcome);
				}
			});
}

function _onFetchError (_task) {
	transcript.traceDebuggingObject ("failed fetching `%s`", _task.url, _task.error);
	_task.callback (_task.error, undefined);
}

// ---------------------------------------

var _hosts420Last = {};
var _hosts420Age = {};

function _fetchUrl (_url, _contentType, _etag, _timestamp, _callback) {
	
	if (_url === undefined)
		throw (new Error ());
	if (_contentType === undefined)
		_contentType = null;
	if (_etag === undefined)
		_etag = null;
	if (_timestamp === undefined)
		_timestamp = null;<commit_msg>Fixed a bug in which fetching errors would go un-noticed.<commit_after>// ---------------------------------------

if (require.main === module)
	throw (new Error ());

// ---------------------------------------

var http = require ("http");
var url = require ("url");

var configuration = require ("./configuration");
var store = require ("./store-lib")
var transcript = require ("./transcript") (module, configuration.libTranscriptLevel);

// ---------------------------------------

function _fetch (_url, _context, _callback) {
	var _task = {};
	_task.url = _url;
	_task.context = _context;
	_task.callback = _callback;
	_task.type = configuration.fetchTaskType;
	_task.feedKey = store.generateFeedKey (_task.url);
	_task.taskKey = store.generateFeedTaskKey (_task.url, _task.type);
	_task.contentType = "application/atom+xml; charset=utf-8";
	_doFetchStep1 (_task);
}

function _doFetchStep1 (_task) {
	transcript.traceDebugging ("fetching `%s` step 1 (fetching latest fetch task)...", _task.url);
	store.fetchFeedTask (_task.context.riak, _task.taskKey,
			function (_error, _previousTaskOutcome, _riakMetaData) {
				if (_error !== null) {
					_task.error = _error;
					_onFetchError (_task);
				} else {
					_task.taskRiakMetaData = _riakMetaData;
					_task.previousTaskOutcome = _previousTaskOutcome;
					if ((_task.previousTaskOutcome !== null)
							&& (
								(_task.previousTaskOutcome.currentEtag === undefined)
								|| (_task.previousTaskOutcome.currentTimestamp === undefined)
								|| (_task.previousTaskOutcome.currentData === undefined)
								|| (_task.previousTaskOutcome.updatedTimestamp === undefined)
								|| (_task.previousTaskOutcome.outcome === undefined)
								|| (_task.previousTaskOutcome.error === undefined)))
						_task.previousTaskOutcome = null;
					if ((_task.previousTaskOutcome === null)
							|| ((new Date () .getTime () - _task.previousTaskOutcome.currentTimestamp) > configuration.fetcherMinFetchAge))
						_doFetchStep2 (_task);
					else {
						_task.error = {reason : "rejected", cause : "fetchAge <="};
						_onFetchError (_task);
					}
				}
			});
}

function _doFetchStep2 (_task) {
	transcript.traceDebugging ("fetching `%s` step 2 (fetching latest data)...", _task.url);
	_fetchUrl (_task.url, _task.contentType,
			_task.previousTaskOutcome ? _task.previousTaskOutcome.previousEtag : null,
			_task.previousTaskOutcome ? _task.previousTaskOutcome.previousTimestamp : null,
			function (_error, _outcome, _data) {
				if (_error !== null) {
					_task.error = _error;
					_task.dataOutcome = null;
					_task.data = null;
					_onFetchStep3b (_task);
				} else {
					_task.dataOutcome = _outcome;
					_task.data = _data;
					if (_task.data !== null)
						_onFetchStep3a (_task);
					else
						_onFetchStep3b (_task);
				}
			});
}

function _onFetchStep3a (_task) {
	transcript.traceDebugging ("fetching `%s` step 3a (updating data)...", _task.url);
	_task.dataKey = store.generateFeedDataKey (_task.url, _task.data);
	store.updateFeedData (_task.context.riak, _task.dataKey, _task.data,
			{
					contentType : _task.dataOutcome.contentType,
					usermeta : {feedKey : _task.feedKey, feedTaskKey : _task.taskKey, feedTaskType : _task.type}},
			function (_error, _riakMetaData) {
				if (_error !== null) {
					_task.error = _error;
					_onFetchError (_task);
				} else {
					_task.dataRiakMetaData = _riakMetaData;
					_onFetchStep4 (_task);
				}
			});
}

function _onFetchStep3b (_task) {
	_task.dataKey = _task.previousTaskOutcome ? _task.previousTaskOutcome.currentData : null;
	_task.dataRiakMetaData = null;
	_onFetchStep4 (_task);
}

function _onFetchStep4 (_task) {
	transcript.traceDebugging ("fetching `%s` step 4 (updating fetch task)...", _task.url);
	_task.currentTaskOutcome = {
		key : _task.taskKey,
		type : _task.type,
		feed : _task.feedKey,
		url : _task.url,
		currentData : _task.dataKey,
		currentEtag : _task.dataOutcome ? _task.dataOutcome.currentEtag : null,
		currentTimestamp : new Date () .getTime (),
		previousData : _task.previousTaskOutcome ? _task.previousTaskOutcome.currentData : null,
		previousEtag : _task.previousTaskOutcome ? _task.previousTaskOutcome.currentEtag : null,
		previousTimestamp : _task.previousTaskOutcome ? _task.previousTaskOutcome.currentTimestamp : null,
		updatedTimestamp : _task.previousTaskOutcome ? _task.previousTaskOutcome.updatedTimestamp : null,
		outcome : _task.dataOutcome,
		error : _task.error ? _task.error : null
	};
	if (_task.currentTaskOutcome.currentData != _task.currentTaskOutcome.previousData)
		_task.currentTaskOutcome.updatedTimestamp = _task.currentTaskOutcome.currentTimestamp;
	_task.taskRiakMetaData.contentType = "application/json";
	store.updateFeedTask (_task.context.riak, _task.taskKey, _task.currentTaskOutcome, _task.taskRiakMetaData,
			function (_error, _riakMetaData) {
				if (_error !== null) {
					_task.error = _error;
					_onFetchError (_task);
				} else {
					_task.taskRiakMetaData = _riakMetaData;
					if (_task.error !== undefined) {
						transcript.traceDebugging ("succeeded fetching `%s`", _task.url);
						_task.callback (null, _task.currentTaskOutcome);
					} else
						_onFetchError (_task);
				}
			});
}

function _onFetchError (_task) {
	transcript.traceDebuggingObject ("failed fetching `%s`", _task.url, _task.error);
	_task.callback (_task.error, undefined);
}

// ---------------------------------------

var _hosts420Last = {};
var _hosts420Age = {};

function _fetchUrl (_url, _contentType, _etag, _timestamp, _callback) {
	
	if (_url === undefined)
		throw (new Error ());
	if (_contentType === undefined)
		_contentType = null;
	if (_etag === undefined)
		_etag = null;
	if (_timestamp === undefined)
		_timestamp = null;
{'fraction_non_alphanumeric': 0.152, 'fraction_numerical': 0.004043956043956044, 'mean_word_length': 3.5, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7461814', 'n_tokens_mistral': 4225, 'n_tokens_neox': 3641, 'n_words': 726}