<commit_before>// # Master
// A simple class to manage a number of worker processes via node's core `cluster` module.
// Supports graceful shutdown via the `SIGTERM` signal, as well as detection child
// processes crashing too quickly after launch.

// [strict mode](http://mzl.la/1fRhnam)
'use strict';

var cluster = require('cluster');
var os = require('os');

/*
The `constructor` has no required parameters.   Optional parameters:

+ `spinTimeout` - how long a process needs to stay up after launch for a crash to be
considered 'normal.'
+ `delayStart` - if a process crashes too fast, how long to wait before starting another
+ `pollInterval` - when gracefully shutting down, how frequently to check for zero
remaining worker processes
+ `killTimeout` - how long to wait after sending 'SIGTERM` to follow up it with a 'SIGINT'
for a more decisive kill
+ `numberWorkers` - worker processes to start and maintain. defaults to `os.cpus().length`

*/
function Master(options) {
  /*jshint maxcomplexity: 10 */

  options = options || {};

  this.spinTimeout = options.spinTimeout || 5000;
  this.delayStart = options.delayStart || 60 * 1000;
  this.pollInterval = options.pollInterval || 500;
  this.killTimeout = options.killTimeout || 7000;
  this.numberWorkers = options.numberWorkers ||
    parseInt(process.env.THEHELP_NUMBER_WORKERS) ||
    os.cpus().length;

  this.workers = {};
  this.closed = false;

  this.cluster = options.cluster || cluster;
  this.cluster.on('disconnect', this._restartWorker.bind(this));

  this.log = options.log || require('winston');
  this.setGraceful(options.graceful);

  Master.instance = this;
}

module.exports = Master;

// `start` gets all the requested worker processes started
Master.prototype.start = function start() {
  this.log.warn('Starting master');
  for (var i = 0; i < this.numberWorkers; i = i + 1) {
    this._startWorker();
  }
};

// `setGraceful` is a way to provide the reference after construction. We register with
// `Graceful` to shutdown when it starts the shutdown process, as well as to let it know
// when workers are still alive. With this in place, you won't need to call `shutdown()`
// or `stop()` yourself.
Master.prototype.setGraceful = function setGraceful(graceful) {
  var _this = this;

  if (graceful) {
    this.graceful = graceful;

    //default timeout for sending SIGINT is longer than timeout for Graceful force kill
    this.graceful.timeout = this.killTimeout + this.pollInterval * 2;

    this.graceful.on('shutdown', function stopWorkers() {
      _this.shutdown();
    });

    this.graceful.addCheck(function areWorkersActive() {
      return !_this.workersActive;
    });
  }
};

// `shutdown` uses `stop` to kill all workers, then sets `this.workersActive` back to
// false so `Graceful` knows that it's safe to take the process down.
Master.prototype.shutdown = function shutdown() {
  var _this = this;

  this.workersActive = true;
  this.stop(function() {
    _this.workersActive = false;
  });
};

// `stop` kills all worker processes, first using a 'SIGTERM' signal to allow for graceful
// shutdown. If the process isn't dead by `this.killTimeout` a 'SIGINT' signal is sent.
Master.prototype.stop = function stop(cb) {
  var _this = this;
  this.log.warn('Stopping all workers with SIGTERM...');

  this.closed = true;
  this._sendToAll('SIGTERM');

  var timeout = setTimeout(function forceKill() {
    timeout = null;
    _this.log.warn('Shutdown delayed; sending SIGINT to all remaining workers...');
    _this._sendToAll('SIGINT');
  }, _this.killTimeout);

  var interval = setInterval(function checkAndReturn() {
    if (!Object.keys(_this.cluster.workers).length) {

      clearInterval(interval);
      if (timeout) {
        clearTimeout(timeout);
      }
<commit_msg>Master: Change SIGTERM log entry to info<commit_after>// # Master
// A simple class to manage a number of worker processes via node's core `cluster` module.
// Supports graceful shutdown via the `SIGTERM` signal, as well as detection child
// processes crashing too quickly after launch.

// [strict mode](http://mzl.la/1fRhnam)
'use strict';

var cluster = require('cluster');
var os = require('os');

/*
The `constructor` has no required parameters.   Optional parameters:

+ `spinTimeout` - how long a process needs to stay up after launch for a crash to be
considered 'normal.'
+ `delayStart` - if a process crashes too fast, how long to wait before starting another
+ `pollInterval` - when gracefully shutting down, how frequently to check for zero
remaining worker processes
+ `killTimeout` - how long to wait after sending 'SIGTERM` to follow up it with a 'SIGINT'
for a more decisive kill
+ `numberWorkers` - worker processes to start and maintain. defaults to `os.cpus().length`

*/
function Master(options) {
  /*jshint maxcomplexity: 10 */

  options = options || {};

  this.spinTimeout = options.spinTimeout || 5000;
  this.delayStart = options.delayStart || 60 * 1000;
  this.pollInterval = options.pollInterval || 500;
  this.killTimeout = options.killTimeout || 7000;
  this.numberWorkers = options.numberWorkers ||
    parseInt(process.env.THEHELP_NUMBER_WORKERS) ||
    os.cpus().length;

  this.workers = {};
  this.closed = false;

  this.cluster = options.cluster || cluster;
  this.cluster.on('disconnect', this._restartWorker.bind(this));

  this.log = options.log || require('winston');
  this.setGraceful(options.graceful);

  Master.instance = this;
}

module.exports = Master;

// `start` gets all the requested worker processes started
Master.prototype.start = function start() {
  this.log.warn('Starting master');
  for (var i = 0; i < this.numberWorkers; i = i + 1) {
    this._startWorker();
  }
};

// `setGraceful` is a way to provide the reference after construction. We register with
// `Graceful` to shutdown when it starts the shutdown process, as well as to let it know
// when workers are still alive. With this in place, you won't need to call `shutdown()`
// or `stop()` yourself.
Master.prototype.setGraceful = function setGraceful(graceful) {
  var _this = this;

  if (graceful) {
    this.graceful = graceful;

    //default timeout for sending SIGINT is longer than timeout for Graceful force kill
    this.graceful.timeout = this.killTimeout + this.pollInterval * 2;

    this.graceful.on('shutdown', function stopWorkers() {
      _this.shutdown();
    });

    this.graceful.addCheck(function areWorkersActive() {
      return !_this.workersActive;
    });
  }
};

// `shutdown` uses `stop` to kill all workers, then sets `this.workersActive` back to
// false so `Graceful` knows that it's safe to take the process down.
Master.prototype.shutdown = function shutdown() {
  var _this = this;

  this.workersActive = true;
  this.stop(function() {
    _this.workersActive = false;
  });
};

// `stop` kills all worker processes, first using a 'SIGTERM' signal to allow for graceful
// shutdown. If the process isn't dead by `this.killTimeout` a 'SIGINT' signal is sent.
Master.prototype.stop = function stop(cb) {
  var _this = this;
  this.log.info('Stopping all workers with SIGTERM...');

  this.closed = true;
  this._sendToAll('SIGTERM');

  var timeout = setTimeout(function forceKill() {
    timeout = null;
    _this.log.warn('Shutdown delayed; sending SIGINT to all remaining workers...');
    _this._sendToAll('SIGINT');
  }, _this.killTimeout);

  var interval = setInterval(function checkAndReturn() {
    if (!Object.keys(_this.cluster.workers).length) {

      clearInterval(interval);
      if (timeout) {
        clearTimeout(timeout);
      }

{'fraction_non_alphanumeric': 0.1176703122941099, 'fraction_numerical': 0.006061404664646199, 'mean_word_length': 3.7705845380263985, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1638790', 'n_tokens_mistral': 2336, 'n_tokens_neox': 2219, 'n_words': 758}