<commit_before>'use strict';

var util = require('util');
var uuid = require('node-uuid');
var fs = require('fs');

var SkaleClient = require('./client.js');
var skale = require('./dataset.js');
var mkdir = require('./mkdir.js');

module.exports = SkaleContext;

util.inherits(SkaleContext, SkaleClient);

function SkaleContext(arg) {
	if (!(this instanceof SkaleContext))
		return new SkaleContext(arg);
	var self = this, worker = this.worker = [];

	arg = arg || {};
	arg.data = arg.data || {};
	arg.data.type = 'master';
	SkaleClient.call(this, arg);
	var inBrowser = (typeof window != 'undefined');
	var maxWorker = process.env.SKALE_MAX_WORKER;

	this.started = this.ended = false;
	this.jobId = 0;
	this.firstData = {};
	this.contextId = uuid.v4();	// context id which will be used as scratch directory name

	this.basedir = '/tmp/skale/' + this.contextId + '/';
	mkdir(this.basedir + 'tmp');
	mkdir(this.basedir + 'stream');

	this.once('connect', function(data) {
		var i;
		if (!maxWorker || maxWorker > data.devices.length)
			maxWorker = data.devices.length;
		for (i = 0; i < maxWorker; i++) {
			self.worker.push(new Worker(data.devices[i]));
			self.firstData[data.devices[i].uuid] = data.devices;
		}
		self.started = true;
		// XXXX: the following line is a bad workaround. Fix mem leaks instead of forcing exit
		if (!inBrowser) process.on('exit', this.end);
	});

	this.on('workerError', function workerError(msg) {
		console.error('Error from worker id %d:', msg.from);<commit_msg>code cleanup<commit_after>'use strict';

var util = require('util');
var uuid = require('node-uuid');
var fs = require('fs');

var SkaleClient = require('./client.js');
var skale = require('./dataset.js');
var mkdir = require('./mkdir.js');

module.exports = SkaleContext;

util.inherits(SkaleContext, SkaleClient);

function SkaleContext(arg) {
	if (!(this instanceof SkaleContext))
		return new SkaleContext(arg);
	var self = this, worker = this.worker = [];

	arg = arg || {};
	arg.data = arg.data || {};
	arg.data.type = 'master';
	SkaleClient.call(this, arg);
	var maxWorker = process.env.SKALE_MAX_WORKER;

	this.started = this.ended = false;
	this.jobId = 0;
	this.firstData = {};
	this.contextId = uuid.v4();	// context id which will be used as scratch directory name

	this.basedir = '/tmp/skale/' + this.contextId + '/';
	mkdir(this.basedir + 'tmp');
	mkdir(this.basedir + 'stream');

	this.once('connect', function(data) {
		var i;
		if (!maxWorker || maxWorker > data.devices.length)
			maxWorker = data.devices.length;
		for (i = 0; i < maxWorker; i++) {
			self.worker.push(new Worker(data.devices[i]));
			self.firstData[data.devices[i].uuid] = data.devices;
		}
		self.started = true;
	});

	this.on('workerError', function workerError(msg) {
		console.error('Error from worker id %d:', msg.from);
{'fraction_non_alphanumeric': 0.17312477781727692, 'fraction_numerical': 0.002132954141485958, 'mean_word_length': 4.2014787430683915, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2918043', 'n_tokens_mistral': 999, 'n_tokens_neox': 948, 'n_words': 234}