<commit_before>#!/usr/bin/env node

// Copyright 2016 Luca-SAS, licensed under the Apache License 2.0

'use strict';

var child_process = require('child_process');
var fs = require('fs');
var os = require('os');
var cluster = require('cluster');
var url = require('url');
var zlib = require('zlib');
var mkdirp = require('mkdirp');
var uuid = require('uuid');
var AWS = require('aws-sdk');

var SkaleClient = require('../lib/client.js');
var Dataset = require('../lib/dataset.js');
var Task = require('../lib/task.js');
var Lines = require('../lib/lines.js');
var sizeOf = require('../lib/rough-sizeof.js');
var readSplit = require('../lib/readsplit.js').readSplit;

//var global = {require: require};

var opt = require('node-getopt').create([
  ['h', 'help', 'print this help text'],
  ['d', 'debug', 'print debug traces'],
  ['m', 'memory=ARG', 'set max memory in MB for workers (default 1024)'],
  ['M', 'MyHost=ARG', 'advertised hostname (peer-to-peer)'],
  ['n', 'nworker=ARG', 'number of workers (default: number of cpus)'],
  ['s', 'slow', 'disable peer-to-peer file transfers though HTTP'],
  ['t', 'tmp=ARG', 'set tmp dirname (default: /tmp)'],
  ['H', 'Host=ARG', 'server hostname (default localhost)'],
  ['P', 'Port=ARG', 'server port (default 12346)'],
  ['V', 'version', 'print version']
]).bindHelp().parseSystem();

if (opt.options.version) {
  var pkg = require('../package');
  return console.log(pkg.name + '-' +  pkg.version);
}

var debug = opt.options.debug || false;
var ncpu = Number(opt.options.nworker) || (process.env.SKALE_WORKER_PER_HOST ? process.env.SKALE_WORKER_PER_HOST : os.cpus().length);
var memory = Number(opt.options.memory || process.env.SKALE_MEMORY || 1024);
var tmp = opt.options.tmp || process.env.SKALE_TMP || '/tmp';
var cgrid;
var mm = new MemoryManager(memory);
var log;
var hostname;
var start = process.hrtime();
ncpu = Number(ncpu);
if (!opt.options.slow)
  hostname = opt.options.MyHost || os.hostname();

if (process.env.SKALE_DEBUG > 1) {
  log = function () {
    var args = Array.prototype.slice.call(arguments);
    var elapsed = process.hrtime(start);
    args.unshift('[worker-controller ' + (elapsed[0] + elapsed[1] / 1e9).toFixed(3) + 's]');
    console.error.apply(null, args);
  };
} else {
  log = function () {};
}
<commit_msg>worker: remove unused tmp<commit_after>#!/usr/bin/env node

// Copyright 2016 Luca-SAS, licensed under the Apache License 2.0

'use strict';

var child_process = require('child_process');
var fs = require('fs');
var os = require('os');
var cluster = require('cluster');
var url = require('url');
var zlib = require('zlib');
var mkdirp = require('mkdirp');
var uuid = require('uuid');
var AWS = require('aws-sdk');

var SkaleClient = require('../lib/client.js');
var Dataset = require('../lib/dataset.js');
var Task = require('../lib/task.js');
var Lines = require('../lib/lines.js');
var sizeOf = require('../lib/rough-sizeof.js');
var readSplit = require('../lib/readsplit.js').readSplit;

//var global = {require: require};

var opt = require('node-getopt').create([
  ['h', 'help', 'print this help text'],
  ['d', 'debug', 'print debug traces'],
  ['m', 'memory=ARG', 'set max memory in MB for workers (default 1024)'],
  ['M', 'MyHost=ARG', 'advertised hostname (peer-to-peer)'],
  ['n', 'nworker=ARG', 'number of workers (default: number of cpus)'],
  ['s', 'slow', 'disable peer-to-peer file transfers though HTTP'],
  ['H', 'Host=ARG', 'server hostname (default localhost)'],
  ['P', 'Port=ARG', 'server port (default 12346)'],
  ['V', 'version', 'print version']
]).bindHelp().parseSystem();

if (opt.options.version) {
  var pkg = require('../package');
  return console.log(pkg.name + '-' +  pkg.version);
}

var debug = opt.options.debug || false;
var ncpu = Number(opt.options.nworker) || (process.env.SKALE_WORKER_PER_HOST ? process.env.SKALE_WORKER_PER_HOST : os.cpus().length);
var memory = Number(opt.options.memory || process.env.SKALE_MEMORY || 1024);
var cgrid;
var mm = new MemoryManager(memory);
var log;
var hostname;
var start = process.hrtime();
ncpu = Number(ncpu);
if (!opt.options.slow)
  hostname = opt.options.MyHost || os.hostname();

if (process.env.SKALE_DEBUG > 1) {
  log = function () {
    var args = Array.prototype.slice.call(arguments);
    var elapsed = process.hrtime(start);
    args.unshift('[worker-controller ' + (elapsed[0] + elapsed[1] / 1e9).toFixed(3) + 's]');
    console.error.apply(null, args);
  };
} else {
  log = function () {};
}

{'fraction_non_alphanumeric': 0.20263334077214906, 'fraction_numerical': 0.01115822361080116, 'mean_word_length': 4.499386503067485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6364524', 'n_tokens_mistral': 1573, 'n_tokens_neox': 1521, 'n_words': 395}