<commit_before>// Ugrid

var net = require('net');
var byline = require('byline');
var thunkify = require('thunkify');

module.exports = function(arg) {
	var host = arg.host;
	var port = arg.port;
	var data = arg.data ? arg.data : {};
	var my_uuid, grid, my_token;
	var pending_cmd = {};
	var grid_events = {};
	var cmd_cnt = 0;

	function send_cmd(json) {
		var cmd = {
			cmd: json.cmd,
			from: my_uuid ? my_uuid : null,
			token: my_token ? my_token : null,
			cmd_id: json.cmd_id ? json.cmd_id : get_next_cmd_id(),
			data: json.data
		}
		if (json.process)
			pending_cmd[cmd.cmd_id] = {process: json.process};
		grid.write(JSON.stringify(cmd) + '\n');
	}

	this.connect_cb = function(callback) {
		grid = net.connect(port, host, function() {
			send('connect', data, function(err, res) {
				if (res) {
					my_uuid = res.uuid;
					my_token = res.token;
				}
				callback(err, res);
			});
			byline(grid).on('data', function(d) {
				try {
					var o = JSON.parse(d, function(key, value) {
						if (value === "gridNaN")
							return NaN;
						return value;
					});
					if (o.data && o.data.err)
						throw o.data.err;
					if (o.cmd in grid_events) {	      // command from server
						grid_events[o.cmd](o);
					} else if (o.cmd_id in pending_cmd) { // answer to previous cmd
						pending_cmd[o.cmd_id].process(o.data.err, o.data.result);
					} 
				} catch (error) {
					console.trace('on received line:\n' + d + '\nError: ' + error);
					process.exit(1);
				}
			});
		});
	};

	this.on = function(event, callback) {
		grid_events[event] = callback;
	};

	this.disconnect = function() {
		grid.end();
	};

	var send = this.send_cb = function(cmd, data, callback) {
		var o = {cmd: cmd, data: data};
		if (o.cmd === 'answer') {
			o.cmd_id = o.data.cmd_id;
		}
		if (callback) {
			o.process = function(error, result, cmd_id) {
				callback(error, result);
			};
		}
		send_cmd(o);
	};

	var get_next_cmd_id = this.get_next_cmd_id = function () {
		if (cmd_cnt++ == 1000000)
			cmd_cnt = 0;
		return cmd_cnt;
	};

	this.connect = thunkify(this.connect_cb);
	this.send = thunkify(this.send_cb);
};<commit_msg>fix memory leak in ugrid-client lib<commit_after>// Ugrid

var assert = require('assert');
var net = require('net');
var byline = require('byline');
var thunkify = require('thunkify');

module.exports = function(arg) {
	var host = arg.host;
	var port = arg.port;
	var data = arg.data ? arg.data : {};
	var my_uuid, grid, my_token;
	var pending_cmd = {};
	var grid_events = {};
	var cmd_cnt = 0;

	function send_cmd(json) {
		var cmd = {
			cmd: json.cmd,
			from: my_uuid ? my_uuid : null,
			token: my_token ? my_token : null,
			cmd_id: json.cmd_id ? json.cmd_id : get_next_cmd_id(),
			data: json.data
		}
		if (json.process)
			pending_cmd[cmd.cmd_id] = {process: json.process};
		grid.write(JSON.stringify(cmd) + '\n');
	}

	this.connect_cb = function(callback) {
		grid = net.connect(port, host, function() {
			send('connect', data, function(err, res) {
				if (res) {
					my_uuid = res.uuid;
					my_token = res.token;
				}
				callback(err, res);
			});
			byline(grid).on('data', function(d) {
				try {
					var o = JSON.parse(d, function(key, value) {
						if (value === "gridNaN")
							return NaN;
						return value;
					});
					if (o.data && o.data.err)
						throw o.data.err;
					if (o.cmd in grid_events) {	      // command from server
						grid_events[o.cmd](o);
					} else if (o.cmd_id in pending_cmd) { // answer to previous cmd
						assert(o.data);
						pending_cmd[o.cmd_id].process(o.data.err, o.data.result);
						delete(pending_cmd[o.cmd_id]);
					} 
				} catch (error) {
					console.trace('on received line:\n' + d + '\nError: ' + error);
					process.exit(1);
				}
			});
		});
	};

	this.on = function(event, callback) {
		grid_events[event] = callback;
	};

	this.disconnect = function() {
		grid.end();
	};

	var send = this.send_cb = function(cmd, data, callback) {
		var o = {cmd: cmd, data: data};
		if (o.cmd === 'answer') {
			o.cmd_id = o.data.cmd_id;
		}
		if (callback) {
			o.process = function(error, result, cmd_id) {
				callback(error, result);
			};
		}
		send_cmd(o);
	};

	var get_next_cmd_id = this.get_next_cmd_id = function () {
		if (cmd_cnt++ == 1000000) {
			console.log("cmd_id overflow!");
			cmd_cnt = 0;
		}
		return cmd_cnt;
	};

	this.connect = thunkify(this.connect_cb);
	this.send = thunkify(this.send_cb);
};
{'fraction_non_alphanumeric': 0.1670061099796334, 'fraction_numerical': 0.004525910839556461, 'mean_word_length': 2.8670166229221348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '614936', 'n_tokens_mistral': 1993, 'n_tokens_neox': 1673, 'n_words': 364}