<commit_before>var ChildProcess = require('child_process');
var JStream = require('jstream');
var fs = require('fs');
var tempfile = require('tempfile');
var debug = require('debug')('streat');

JStream.MAX_BUFFER_LENGTH = 1024 * 1024;

module.exports = Streat;

function Streat(opts) {
	this.step = (opts || {}).step || 32768;
	this.running = false;
	this.filepath = tempfile();
	this.service = null;
	this.queue = [];
}

Streat.prototype.run = function(res, limit, cb) {
	if (!cb && typeof limit == "function") {
		cb = limit;
		limit = 0;
	}
	this.queue.push({
		res: res,
		limit: limit,
		cb: cb,
		begin: 0,
		size: 0
	});
	if (this.queue.length == 1) setImmediate(this.next.bind(this));
};

Streat.prototype.start = function() {
	if (this.running) return;

	debug('spawn exiftool');
	this.running = true;

	this.service = ChildProcess.spawn('exiftool', [
		'-stay_open', 'True', '-@', '-'
	]);

	this.service.on('exit', function(code, signal) {
		this.running = false;
		debug('exiftool exits with code', code, 'and signal', signal);
	}.bind(this));

	this.service.stderr.on('data', function(data) {
		if (data && data.length) console.error("error", data.toString());
	});

	this.service.stdout.pipe(new JStream()).on('data', function(tags) {
		debug("exiftool got", tags);
		this.done(null, tags); // err should be collected above
		this.next();
	}.bind(this)).on('error', function(err) {
		debug("exiftool unparseable reply", err);
		this.done(err);
		this.next();
	}.bind(this));
};

Streat.prototype.send = function(cmds) {
	debug("sending command to exiftool");
	this.service.stdin.cork();
	this.service.stdin.write('-q\n' + cmds.join('\n') + '\n-execute\n');
	this.service.stdin.uncork();
};

Streat.prototype.stop = function() {
	this.send(['-stay_open','False']);
	this.service.kill();
	this.running = false;
	this.service = null;
};

Streat.prototype.done = function(err, tags) {
	var runner = this.queue[0];
	if (!runner) return console.error("missing runner");
	if (!tags) {
		if (!err) err = new Error("exiftool returned no tags");
	}<commit_msg>Make sure spawned exiftool is killed on exit<commit_after>var ChildProcess = require('child_process');
var JStream = require('jstream');
var fs = require('fs');
var tempfile = require('tempfile');
var debug = require('debug')('streat');

JStream.MAX_BUFFER_LENGTH = 1024 * 1024;

module.exports = Streat;

function Streat(opts) {
	this.step = (opts || {}).step || 32768;
	this.running = false;
	this.filepath = tempfile();
	this.service = null;
	this.queue = [];
}

Streat.prototype.run = function(res, limit, cb) {
	if (!cb && typeof limit == "function") {
		cb = limit;
		limit = 0;
	}
	this.queue.push({
		res: res,
		limit: limit,
		cb: cb,
		begin: 0,
		size: 0
	});
	if (this.queue.length == 1) setImmediate(this.next.bind(this));
};

Streat.prototype.start = function() {
	if (this.running) return;

	debug('spawn exiftool');
	this.running = true;

	process.on('exit', function() {
		if (this.service) this.stop();
	}.bind(this));

	this.service = ChildProcess.spawn('exiftool', [
		'-stay_open', 'True', '-@', '-'
	]);

	this.service.on('exit', function(code, signal) {
		this.running = false;
		debug('exiftool exits with code', code, 'and signal', signal);
	}.bind(this));

	this.service.stderr.on('data', function(data) {
		if (data && data.length) console.error("error", data.toString());
	});

	this.service.stdout.pipe(new JStream()).on('data', function(tags) {
		debug("exiftool got", tags);
		this.done(null, tags); // err should be collected above
		this.next();
	}.bind(this)).on('error', function(err) {
		debug("exiftool unparseable reply", err);
		this.done(err);
		this.next();
	}.bind(this));
};

Streat.prototype.send = function(cmds) {
	debug("sending command to exiftool");
	this.service.stdin.cork();
	this.service.stdin.write('-q\n' + cmds.join('\n') + '\n-execute\n');
	this.service.stdin.uncork();
};

Streat.prototype.stop = function() {
	this.send(['-stay_open','False']);
	this.service.kill('SIGKILL');
	this.running = false;
	this.service = null;
};

Streat.prototype.done = function(err, tags) {
	var runner = this.queue[0];
	if (!runner) return console.error("missing runner");
	if (!tags) {
		if (!err) err = new Error("exiftool returned no tags");
	}
{'fraction_non_alphanumeric': 0.19915353867857982, 'fraction_numerical': 0.00846461321420174, 'mean_word_length': 4.426020408163265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1299245', 'n_tokens_mistral': 1601, 'n_tokens_neox': 1513, 'n_words': 329}