<commit_before>'use strict';
/*jshint latedef:false */

/* Setting up server process and client process, which gives ~8K concurrent
sysctl -a | grep maxfiles  # display maxfiles and maxfilesperproc  defaults 12288 and 10240
sudo sysctl -w kern.maxfiles=25000
sudo sysctl -w kern.maxfilesperproc=24500
sysctl -a | grep somax # display max socket setting, default 128
sudo sysctl -w kern.ipc.somaxconn=20000  # set
ulimit -S -n       # display soft max open files, default 256
ulimit -H -n       # display hard max open files, default unlimited
ulimit -S -n 20000  # set soft max open files
 */

var async = require('async');
var http = require('http');
var measured = require('measured');
var request = require('request');
var EventEmitter = require('events').EventEmitter;

function benchmark(flow, runOptions) {
  if (!runOptions || !runOptions.iterations || !runOptions.limit) {
    throw new Error('benchmark runOptions requires iterations and limit properties');
  }
  flow = ensureFlowProperties(flow);
  var emitter = new EventEmitter();
  var errorCount = 0;
  var stats = measured.createCollection();
  var elapsed_timer = stats.timer('totalElapsed').start();

  function emitProgress() {
    var statsObj = stats.toJSON();
    var percent = (statsObj.main) ? Math.round(statsObj.main.meter.count * 100 / runOptions.iterations) : 0;
    emitter.emit('progress', statsObj, percent);
  }
  var progressTimer = (runOptions.progress) ? setInterval(emitProgress, runOptions.progress) : null;

  function handleError(err, ctxName, exit) {
    errorCount++;
    emitter.emit('error', err, ctxName);
    if (exit) emitter.emit('end', stats, errorCount);
  }

  http.globalAgent.maxSockets = runOptions.limit;

  function run(task, cb) {
    var req_timer = stats.timer('main').start();
    task.env.group = 'beforeMain';
    performActions(task.env, task.beforeMain, function (err) {
      if (err) {
        handleError(err, 'beforeMain');
        return cb(err);
      }
      task.env.group = 'main';
      performActions(task.env, task.main, function (err) {
        req_timer.end();
        if (err) {
          handleError(err, 'main');
        }
        task.env.group = 'afterMain';
        performActions(task.env, task.afterMain, function (err) {
          if (err) {
            handleError(err, 'afterMain');
            return cb(err);
          }
          cb();
        });
      });
    });
  }


  var queue = async.queue(run, runOptions.limit);

  queue.drain = function () {
    performActions({ group: 'after', user: runOptions.user, password: <PASSWORD>  }, flow.after, function (err) {
      elapsed_timer.end();
      if (err) {
        handleError(err, 'after');
      }
      if (progressTimer) clearInterval(progressTimer);
      var statsObj = stats.toJSON(); // actually materializes object
      statsObj.totalElapsed = statsObj.totalElapsed.histogram.max; // simplify the elapsed to single number
      emitter.emit('end', statsObj, errorCount);
    });
  };

  process.nextTick(function () { // allow event handlers to be hooked up before starting

    performActions({ group: 'before', user: runOptions.user, password: <PASSWORD> }, flow.before, function (err) {
      if (err) {
        return handleError(err, 'before', true);  // exit if fails here
      }

      try {
        for (var i = 0; i < runOptions.iterations; i++) {
          var tokens = { INDEX: i };
          queue.push({
            env: { index: i, jar: request.jar(), user: runOptions.user, password: <PASSWORD>, etags: {} },<commit_msg>call stats.end() to cause internal cleanup of its timers<commit_after>'use strict';
/*jshint latedef:false */

/* Setting up server process and client process, which gives ~8K concurrent
sysctl -a | grep maxfiles  # display maxfiles and maxfilesperproc  defaults 12288 and 10240
sudo sysctl -w kern.maxfiles=25000
sudo sysctl -w kern.maxfilesperproc=24500
sysctl -a | grep somax # display max socket setting, default 128
sudo sysctl -w kern.ipc.somaxconn=20000  # set
ulimit -S -n       # display soft max open files, default 256
ulimit -H -n       # display hard max open files, default unlimited
ulimit -S -n 20000  # set soft max open files
 */

var async = require('async');
var http = require('http');
var measured = require('measured');
var request = require('request');
var EventEmitter = require('events').EventEmitter;

function benchmark(flow, runOptions) {
  if (!runOptions || !runOptions.iterations || !runOptions.limit) {
    throw new Error('benchmark runOptions requires iterations and limit properties');
  }
  flow = ensureFlowProperties(flow);
  var emitter = new EventEmitter();
  var errorCount = 0;
  var stats = measured.createCollection();
  var elapsed_timer = stats.timer('totalElapsed').start();

  function emitProgress() {
    var statsObj = stats.toJSON();
    var percent = (statsObj.main) ? Math.round(statsObj.main.meter.count * 100 / runOptions.iterations) : 0;
    emitter.emit('progress', statsObj, percent);
  }
  var progressTimer = (runOptions.progress) ? setInterval(emitProgress, runOptions.progress) : null;

  function handleError(err, ctxName, exit) {
    errorCount++;
    emitter.emit('error', err, ctxName);
    if (exit) {
      shutdown();
      emitter.emit('end', stats, errorCount);
    }
  }

  http.globalAgent.maxSockets = runOptions.limit;

  function run(task, cb) {
    var req_timer = stats.timer('main').start();
    task.env.group = 'beforeMain';
    performActions(task.env, task.beforeMain, function (err) {
      if (err) {
        handleError(err, 'beforeMain');
        return cb(err);
      }
      task.env.group = 'main';
      performActions(task.env, task.main, function (err) {
        req_timer.end();
        if (err) {
          handleError(err, 'main');
        }
        task.env.group = 'afterMain';
        performActions(task.env, task.afterMain, function (err) {
          if (err) {
            handleError(err, 'afterMain');
            return cb(err);
          }
          cb();
        });
      });
    });
  }

  function shutdown() {
    if (progressTimer) {
      clearInterval(progressTimer);
      progressTimer = null;
    }
    stats.end();
  }

  var queue = async.queue(run, runOptions.limit);

  queue.drain = function () {
    performActions({ group: 'after', user: runOptions.user, password: <PASSWORD>  }, flow.after, function (err) {
      elapsed_timer.end();
      if (err) {
        handleError(err, 'after');
      }
      shutdown();
      var statsObj = stats.toJSON(); // actually materializes object
      statsObj.totalElapsed = statsObj.totalElapsed.histogram.max; // simplify the elapsed to single number
      emitter.emit('end', statsObj, errorCount);
    });
  };

  process.nextTick(function () { // allow event handlers to be hooked up before starting

    performActions({ group: 'before', user: runOptions.user, password: <PASSWORD> }, flow.before, function (err) {
      if (err) {
        return handleError(err, 'before', true);  // exit if fails here
      }

      try {
        for (var i = 0; i < runOptions.iterations; i++) {
          var tokens = { INDEX: i };
          queue.push({
            env: { index: i, jar: request.jar(), user: runOptions.user, password: <PASSWORD>, etags: {} },
{'fraction_non_alphanumeric': 0.1315426997245179, 'fraction_numerical': 0.011845730027548209, 'mean_word_length': 3.018262313226342, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5065464', 'n_tokens_mistral': 2264, 'n_tokens_neox': 2132, 'n_words': 609}