<commit_before>var express = require('express');
var path = require('path');
var url = require('url');

var app = express();

var server = app.listen(process.env.PORT || 3000, function () {
	console.log('Listening on port %d', server.address().port);
});

var timeValue = 0;

setInterval(function () {
	timeValue = Math.floor(Date.now() / 1000);
}, 100);

var getTime = function () {
	return timeValue;
};

var headerHandler = function (value, req, res) {
	var etag = '"' + value + '"';
	res.set('ETag', etag);

	var allowHeaders = ['If-None-Match', 'Wait'];
	var exposeHeaders = [];

	if(!req.query.no_poll_header) {
		res.set('X-Poll-Interval', '10');
		exposeHeaders.push('X-Poll-Interval');
	}

	var links = [];
	links.push('</test?after=' + value + '>; rel=changes');

	if(links.length > 0) {
		res.set('Link', links.join(', '));
		exposeHeaders.push('Link');
	}

	// cors
	res.set('Access-Control-Max-Age', '3600');
	res.set('Access-Control-Allow-Origin', req.get('Origin') || '*');
	if(allowHeaders.length > 0) {
		res.set('Access-Control-Allow-Headers', allowHeaders.join(', '));
	}
	if(exposeHeaders.length > 0) {
		res.set('Access-Control-Expose-Headers', exposeHeaders.join(', '));
	}

	return etag;
};

app.head('/test', function (req, res) {
	var value = getTime();
	headerHandler(value, req, res);
	res.send('')
});

app.get('/test', function (req, res) {
	var value = getTime();
	var etag = headerHandler(value, req, res);

	if(req.query.after) {
		var after = parseInt(req.query.after);
		if(after > value) {
			// went back in time? tell the client to start over
			res.status(404).end();
			return;
		}
		res.status(200).json({'time:change': '+' + (value - after)});
	} else {
		var inm = req.get('If-None-Match');
		if (inm == etag) {
			res.status(304).end();
		} else {
			res.status(200).json({time: value});
		}
	}
});<commit_msg>support stream push<commit_after>var express = require('express');
var path = require('path');
var url = require('url');

var app = express();

var server = app.listen(process.env.PORT || 3000, function () {
	console.log('Listening on port %d', server.address().port);
});

var listeners = [];

var timeValue = 0;

setInterval(function () {
	var old = timeValue;
	timeValue = Math.floor(Date.now() / 1000);

	if(timeValue != old) {
		for(var i = 0; i < listeners.length; ++i) {
			var l = listeners[i];
			if(l.type == 'hint') {
				l.res.write('event: update\ndata:\n\n');
			} else if(l.type == 'changes') {
				var meta = {
					'Content-Type': 'application/json',
					'Changes-Id': '' + timeValue,
					'Previous-Changes-Id': '' + old
				};
				l.res.write('event: update\n' +
					'data: ' + JSON.stringify(meta) + '\n' +
					'data: ' + JSON.stringify({'time:change': '+' + (timeValue - old)}) + '\n\n');
			} else {
				var meta = {
					'Content-Type': 'application/json',
					'ETag': '"' + timeValue + '"',
					'Previous-ETag': '"' + old + '"'
				};
				l.res.write('event: update\n' +
					'data: ' + JSON.stringify(meta) + '\n' +
					'data: ' + JSON.stringify({time: timeValue}) + '\n\n');
			}
		}
	}
}, 100);

var getTime = function () {
	return timeValue;
};

var headerHandler = function (value, req, res) {
	var etag = '"' + value + '"';
	res.set('ETag', etag);

	var allowHeaders = ['If-None-Match', 'Wait'];
	var exposeHeaders = [];
	exposeHeaders.push('ETag');
	exposeHeaders.push('Previous-ETag');
	exposeHeaders.push('Changes-Id');
	exposeHeaders.push('Previous-Changes-Id');

	if('no_poll_header' in req.query) {
		res.set('X-Poll-Interval', '10');
		exposeHeaders.push('X-Poll-Interval');
	}

	var links = [];
	links.push('</test?after=' + value + '>; rel=changes');
	links.push('</test>; rel=value-stream');
	links.push('</test?changes>; rel=changes-stream');
	links.push('</test?hints>; rel=hint-stream');

	if(links.length > 0) {
		res.set('Link', links.join(', '));
		exposeHeaders.push('Link');
	}

	res.set('Changes-Id', '' + value);

	// cors
	res.set('Access-Control-Max-Age', '3600');
	res.set('Access-Control-Allow-Origin', req.get('Origin') || '*');
	if(allowHeaders.length > 0) {
		res.set('Access-Control-Allow-Headers', allowHeaders.join(', '));
	}
	if(exposeHeaders.length > 0) {
		res.set('Access-Control-Expose-Headers', exposeHeaders.join(', '));
	}

	return etag;
};

app.head('/test', function (req, res) {
	var value = getTime();
	headerHandler(value, req, res);
	res.send('')
});

app.get('/test', function (req, res) {
	var value = getTime();
	var etag = headerHandler(value, req, res);

	var sse = false;
	var accept = req.get('Accept');
	if(accept) {
		// do SSE only if mime type explicitly specified
		accept = accept.split(',');
		for(var i = 0; i < accept.length; ++i) {
			if(accept[i].split(';')[0].trim() == 'text/event-stream') {
				sse = true;
				break;
			}
		}
	}

	if(sse) {
		var type;
		if('hints' in req.query) {
			type = 'hint';
		} else if('changes' in req.query) {
			type = 'changes';
		} else {
			type = 'value';
		}

		var l = {type: type, res: res};
		listeners.push(l);
		req.on('close', function () {
			var i = listeners.indexOf(l);
			listeners.splice(i, 1);
		});

		res.set('Content-Type', 'text/event-stream');
		res.write('event: open\ndata:\n\n');
		return;
	}

	if('after' in req.query) {
		var after = parseInt(req.query.after);
		if(after > value) {
			// went back in time? tell the client to start over
			res.status(404).end();
			return;
		}
		res.status(200).send(JSON.stringify({'time:change': '+' + (value - after)}) + '\n');
	} else {
		var inm = req.get('If-None-Match');
		if (inm == etag) {
			res.status(304).end();
		} else {
			res.status(200).send(JSON.stringify({time: value}) + '\n');
		}
	}
});
{'fraction_non_alphanumeric': 0.22131872016970125, 'fraction_numerical': 0.01237404984974368, 'mean_word_length': 3.5813765182186237, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1935403', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2098, 'n_words': 415}