<commit_before>function Downloader() {
	this.isActive = false;
	this.realtime = false;
	this.chunkStart = 0;
	this.chunkSize = 0;
	this.totalLength = 0;
	this.chunkTimeout = 1000;
	this.url = null;
	this.callback = null;
}

Downloader.prototype.reset = function() {
	this.chunkStart = 0;
	this.totalLength = 0;
}

Downloader.prototype.setRealTime = function(_realtime) {
	this.realtime = _realtime;
}

Downloader.prototype.setChunkSize = function(_size) {
	this.chunkSize = _size;
}

Downloader.prototype.setChunkStart = function(_start) {
	this.chunkStart = _start;
}

Downloader.prototype.setInterval = function(_timeout) {
	this.chunkTimeout = _timeout;
}

Downloader.prototype.setUrl = function(_url) {
	this.url = _url;
}

Downloader.prototype.setCallback = function(_callback) {
	this.callback = _callback;
}

Downloader.prototype.isStopped = function () {
	return !this.isActive;
}

Downloader.prototype.getFileLength = function () {
	return this.totalLength;
}

Downloader.prototype.getFile = function() {
	var dl = this;
	if (this.chunkStart === Infinity) {
		Log.i("Downloader", "File download done.");
		this.callback(null, true);
		return;
	}
	var xhr = new XMLHttpRequest();
	xhr.open("GET", this.url, true);
	xhr.responseType = "arraybuffer";
	var range = null;
	xhr.start = this.chunkStart;
	if (this.chunkStart+this.chunkSize < Infinity) {
		range = 'bytes=' + this.chunkStart + '-';
		range += (this.chunkStart+this.chunkSize-1);
		xhr.setRequestHeader('Range', range);
	}
	xhr.onerror = function(e) {
		dl.callback(null, false, true);
	}
	xhr.onreadystatechange = function (e) { 
		if ((xhr.status == 200 || xhr.status == 206 || xhr.status == 304 || xhr.status == 416) && xhr.readyState == this.DONE) {
			var rangeReceived = xhr.getResponseHeader("Content-Range");
			Log.i("Downloader", "Received data range: "+rangeReceived);
			if (!dl.totalLength) {
				var sizeIndex;
				sizeIndex = rangeReceived.indexOf("/");
				if (sizeIndex > -1) {
					dl.totalLength = +rangeReceived.slice(sizeIndex+1);
				}
			}
			var eof = (xhr.response.byteLength !== dl.chunkSize);
			xhr.response.fileStart = xhr.start;<commit_msg>clamp byte range request to file size in demo app<commit_after>function Downloader() {
	this.isActive = false;
	this.realtime = false;
	this.chunkStart = 0;
	this.chunkSize = 0;
	this.totalLength = 0;
	this.chunkTimeout = 1000;
	this.url = null;
	this.callback = null;
}

Downloader.prototype.reset = function() {
	this.chunkStart = 0;
	this.totalLength = 0;
}

Downloader.prototype.setRealTime = function(_realtime) {
	this.realtime = _realtime;
}

Downloader.prototype.setChunkSize = function(_size) {
	this.chunkSize = _size;
}

Downloader.prototype.setChunkStart = function(_start) {
	this.chunkStart = _start;
}

Downloader.prototype.setInterval = function(_timeout) {
	this.chunkTimeout = _timeout;
}

Downloader.prototype.setUrl = function(_url) {
	this.url = _url;
}

Downloader.prototype.setCallback = function(_callback) {
	this.callback = _callback;
}

Downloader.prototype.isStopped = function () {
	return !this.isActive;
}

Downloader.prototype.getFileLength = function () {
	return this.totalLength;
}

Downloader.prototype.getFile = function() {
	var dl = this;
	if (this.chunkStart === Infinity) {
		Log.i("Downloader", "File download done.");
		this.callback(null, true);
		return;
	}
	var xhr = new XMLHttpRequest();
	xhr.open("GET", this.url, true);
	xhr.responseType = "arraybuffer";
	var range = null;
	xhr.start = this.chunkStart;
	var maxRange;
	if (this.chunkStart+this.chunkSize < Infinity) {
		range = 'bytes=' + this.chunkStart + '-';
		maxRange = this.chunkStart+this.chunkSize-1;
		/* if the file length is known we limit the max range to that length */
		/*if (dl.totalLength !== 0) {
			maxRange = Math.min(maxRange, dl.totalLength);
		}*/
		range += maxRange;
		xhr.setRequestHeader('Range', range);
	}
	xhr.onerror = function(e) {
		dl.callback(null, false, true);
	}
	xhr.onreadystatechange = function (e) { 
		if ((xhr.status == 200 || xhr.status == 206 || xhr.status == 304 || xhr.status == 416) && xhr.readyState == this.DONE) {
			var rangeReceived = xhr.getResponseHeader("Content-Range");
			Log.i("Downloader", "Received data range: "+rangeReceived);
			/* if the length of the file is not known, we get it from the response header */
			if (!dl.totalLength) {
				var sizeIndex;
				sizeIndex = rangeReceived.indexOf("/");
				if (sizeIndex > -1) {
					dl.totalLength = +rangeReceived.slice(sizeIndex+1);
				}
			}
			var eof = (xhr.response.byteLength !== dl.chunkSize);
			xhr.response.fileStart = xhr.start;
{'fraction_non_alphanumeric': 0.14699477351916376, 'fraction_numerical': 0.010670731707317074, 'mean_word_length': 4.143337066069429, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5130637', 'n_tokens_mistral': 1700, 'n_tokens_neox': 1571, 'n_words': 326}