<commit_before>// a web worker for use by the renderer

var data;
var currentXHRs;
var previousXHRs;
var annotationsXHR;
var sourcesPerDataServer;

var NO_DATA_FOR_TIMESTAMP = "nodata";

self.onmessage = function(evt) {
	data = evt.data;
	data.end = Math.floor(Date.now() / 1000);
	data.start = data.end - data.zoom;

	// we treat the value 1 specially, since it depends on zoom level
	// and we prefer it to be non-zero for evaluation in boolean contexts
	if (data.options.overlay_previous_period == 1) {
		data.offset = data.zoom;
	} else {
		// this'll be null or some # of seconds
		data.offset = data.options.overlay_previous_period;
	}

	var i = 0;
	// cancel any ongoing requests
	if (currentXHRs) {
		for (i = 0; i < currentXHRs.length; i++) {
			currentXHRs[i].abort();
		}
	}
	if (previousXHRs) {
		for (i=0; i < previousXHRs.length; i++) {
			previousXHRs[i].abort();
		}
	}
	currentXHRs = {};
	previousXHRs = {};
	annotationsXHR && annotationsXHR.abort();

	sourcesPerDataServer = {};

	var dataServer;

	for (var sourceIndex = 0; sourceIndex < data.sources.length; sourceIndex++) {
		var source = data.sources[sourceIndex];
		dataServer = source[0];
		if (!sourcesPerDataServer[dataServer]) {
			sourcesPerDataServer[dataServer] = {};
		}
		sourcesPerDataServer[dataServer][sourceIndex] = source;
	}

	for (dataServer in sourcesPerDataServer) {
		var dataServerSources = [];
		var sortedDataServerPosKeys = Object.keys(sourcesPerDataServer[dataServer]).sort();
		for (var _posKey = 0; _posKey < sortedDataServerPosKeys.length; _posKey++) {
			var posKey = sortedDataServerPosKeys[_posKey];
			dataServerSources.push(sourcesPerDataServer[dataServer][posKey]);
		}
		// start our new request(s)
		currentXHRs[dataServer] = fetchData(dataServer, dataServerSources, data.start, data.end);
		if (data.options.overlay_previous_period) {
			previousXHRs[dataServer] = fetchData(dataServer, dataServerSources, data.start - data.offset, data.end - data.offset);
		}
	}
	if (data.options.show_annotations) {
		// TODO (fhats): Pull annotations from all the different data servers, not just one
		// TODO (fhats): Annotations shouldn't require any knowledge of the sources.
		annotationsXHR = fetchAnnotations(data.sources[0][0], data.sources, data.start, data.end);
	}
};

function fetchData(dataServer, sources, start, end) {
	var xhr = new XMLHttpRequest();
	var url = dataServer + "/data?" +
		"sources=" + encodeURIComponent(JSON.stringify(sources.map(function (x){ return x.slice(1); }))) +
		"&start="  + (start - 60) + // buffer for one minute
		"&end="    + end +
		"&width="  + data.width +
		"&token="  + data.token;

	xhr.open("GET", url, true);
	xhr.onreadystatechange = handleResponse;
	xhr.send(null);
	return xhr;
}

function fetchAnnotations(dataServer, sources, start, end) {
	var xhr = new XMLHttpRequest();
	var url = dataServer + "/annotations?" +
		"sources=" + encodeURIComponent(JSON.stringify(sources.map(function (x){ return x.slice(1); }))) +
		"&start="  + (start - 60) + // buffer for one minute
		"&end="    + end +
		"&width="  + data.width +
		"&token="  + data.token;

	xhr.open("GET", url, true);
	xhr.onreadystatechange = handleResponse;
	xhr.send(null);
	return xhr;
}

/**
 * Tells authoritatively whether or not all outstanding XHRs are complete.
 */<commit_msg>Changed js file to remove unnecessary GET parameters<commit_after>// a web worker for use by the renderer

var data;
var currentXHRs;
var previousXHRs;
var annotationsXHR;
var sourcesPerDataServer;

var NO_DATA_FOR_TIMESTAMP = "nodata";

self.onmessage = function(evt) {
	data = evt.data;
	data.end = Math.floor(Date.now() / 1000);
	data.start = data.end - data.zoom;

	// we treat the value 1 specially, since it depends on zoom level
	// and we prefer it to be non-zero for evaluation in boolean contexts
	if (data.options.overlay_previous_period == 1) {
		data.offset = data.zoom;
	} else {
		// this'll be null or some # of seconds
		data.offset = data.options.overlay_previous_period;
	}

	var i = 0;
	// cancel any ongoing requests
	if (currentXHRs) {
		for (i = 0; i < currentXHRs.length; i++) {
			currentXHRs[i].abort();
		}
	}
	if (previousXHRs) {
		for (i=0; i < previousXHRs.length; i++) {
			previousXHRs[i].abort();
		}
	}
	currentXHRs = {};
	previousXHRs = {};
	annotationsXHR && annotationsXHR.abort();

	sourcesPerDataServer = {};

	var dataServer;

	for (var sourceIndex = 0; sourceIndex < data.sources.length; sourceIndex++) {
		var source = data.sources[sourceIndex];
		dataServer = source[0];
		if (!sourcesPerDataServer[dataServer]) {
			sourcesPerDataServer[dataServer] = {};
		}
		sourcesPerDataServer[dataServer][sourceIndex] = source;
	}

	for (dataServer in sourcesPerDataServer) {
		var dataServerSources = [];
		var sortedDataServerPosKeys = Object.keys(sourcesPerDataServer[dataServer]).sort();
		for (var _posKey = 0; _posKey < sortedDataServerPosKeys.length; _posKey++) {
			var posKey = sortedDataServerPosKeys[_posKey];
			dataServerSources.push(sourcesPerDataServer[dataServer][posKey]);
		}
		// start our new request(s)
		currentXHRs[dataServer] = fetchData(dataServer, dataServerSources, data.start, data.end);
		if (data.options.overlay_previous_period) {
			previousXHRs[dataServer] = fetchData(dataServer, dataServerSources, data.start - data.offset, data.end - data.offset);
		}
	}
	if (data.options.show_annotations) {
		// TODO (fhats): Pull annotations from all the different data servers, not just one
		// TODO (fhats): Annotations shouldn't require any knowledge of the sources.
		annotationsXHR = fetchAnnotations(data.sources[0][0], data.sources, data.start, data.end);
	}
};

function fetchData(dataServer, sources, start, end) {
	var xhr = new XMLHttpRequest();
	var url = dataServer + "/data?" +
		"sources=" + encodeURIComponent(JSON.stringify(sources.map(function (x){ return x.slice(1); }))) +
		"&start="  + (start - 60) + // buffer for one minute
		"&end="    + end +
		"&width="  + data.width +
		"&token="  + data.token;

	xhr.open("GET", url, true);
	xhr.onreadystatechange = handleResponse;
	xhr.send(null);
	return xhr;
}

function fetchAnnotations(dataServer, sources, start, end) {
	var xhr = new XMLHttpRequest();
	var url = dataServer + "/annotations?" +
		"start="  + (start - 60) + // buffer for one minute
		"&end="    + end +
		"&token="  + data.token;

	xhr.open("GET", url, true);
	xhr.onreadystatechange = handleResponse;
	xhr.send(null);
	return xhr;
}

/**
 * Tells authoritatively whether or not all outstanding XHRs are complete.
 */
{'fraction_non_alphanumeric': 0.13803162904959312, 'fraction_numerical': 0.005988023952095809, 'mean_word_length': 4.248992747784045, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4423291', 'n_tokens_mistral': 2329, 'n_tokens_neox': 2192, 'n_words': 550}