<commit_before>/*jshint node:true, strict:false*/

// TODO: Use update-notifier to encourage use of latest version of nbob
'use strict';

// Monkey patch fs to work around "EMFILE, too many open files"
require('graceful-fs');

var assert = require('assert');
var _ = require('lodash');
var Promise = require('./promise');
var pkg = require('../package.json');
var commands = require('./commands');
var args = require('./args');
var config = require('./config');
var Logger = require('./logger');
var files = require('./files');
var Processor = require('./processor');
var Schedule = require('./schedule');
var bar = require('./bar');
var log = Logger.create('nbob');

function timeSince(start) {
	var ms = Date.now() - start;
	return ms < 100 ? ms + 'ms' : (ms / 1000).toFixed(1) + 's';
}

var argCommands = args.commands;
if (argCommands.length === 0) {
	// TODO: Show brief help here, add option --help for showing full help
	log.help('No command(s) specified');
}

log.info('nBob v%s: %s v%s', pkg.version, config.project.name, config.project.version);
log.debug('Commands:', argCommands.map(function(command) {
	return commands.getName(command);
}).join(', '));
log.debug('Options:', args.options);
log.spam('Config:', config);

process.chdir(args.options.dir);

files.init();
log.spam('Files:\n  %s', files.getPaths().join('\n  '));

var schedule = new Schedule();
var readOnlyPromises = [];
_.filter(commands.select(argCommands), 'processor').forEach(function(command) {
	var processorName = command.processor;
	var commandName = commands.getName(command);
	var shortName = commands.getName(command, true);
	var pConfig = _.extend(config[commandName], { project: config.project });
	var pLog = Logger.create(shortName);
	var processorStart;

	var processor = require('./processor/' + processorName + '.js');
	if (processor instanceof Processor) {
		processor.init(pConfig, pLog);

		var batches = processor.getBatches(files.getPaths());

		var batchesLen = batches.length;
		var onFinishBatch = _.after(batchesLen, function onFinishBatches() {
			pLog.resume();
			pLog.info('Finished %d batches after: %s', batchesLen, timeSince(processorStart));
		});

		batches.forEach(function(batch, i) {
			var isReadOnly = batch.isReadOnly();

			bar.add(shortName);

			// TODO: Optimize by dividing across multiple cores using child_process.fork()
			// I'm thinking per batch, but sending file data back and forth to children might be inefficient.
			// Unfortunately, looking for shared memory modules I did not find anything good and cross platform.
			// Alternatively we could simply just fork read-only processors like jshint.
			schedule.addParallel(function() {
				var batchStart = Date.now();
				if (!processorStart) {
					processorStart = batchStart;
					pLog.debug('Start %d batches', batchesLen);
					pLog.pause();
				}

				var inputFiles = files.getList(batch.inputPaths);
				var promise = processor.process(inputFiles, batch.outputPaths).then(function(outputFiles) {
					if (isReadOnly) {
						assert.strictEqual(outputFiles, undefined, 'Read only processor should return no output');
					} else {
						var outputPaths = _.pluck(outputFiles, 'path').sort();
						var batchOutputPaths = batch.outputPaths.slice().sort(); // slice to work around Object.freeze
						assert.deepEqual(outputPaths, batchOutputPaths, 'Processor output should match batch');

						files.process(inputFiles, outputFiles);
					}

					bar.finish(shortName);
					pLog.spam('Finished batch #%d after: %s', i + 1, timeSince(batchStart));
					onFinishBatch();
				}, function(error) {
					pLog.info('Rejected batch #%d after: %s', i + 1, timeSince(batchStart));
					onFinishBatch();
					return Promise.reject(error);
				});
				if (isReadOnly) {
					readOnlyPromises.push(promise);
					return Promise.resolve();
				}
				return promise;
			});<commit_msg>Got rid of useless batchStart variable (always practically equal to processorStart)<commit_after>/*jshint node:true, strict:false*/

// TODO: Use update-notifier to encourage use of latest version of nbob
'use strict';

// Monkey patch fs to work around "EMFILE, too many open files"
require('graceful-fs');

var assert = require('assert');
var _ = require('lodash');
var Promise = require('./promise');
var pkg = require('../package.json');
var commands = require('./commands');
var args = require('./args');
var config = require('./config');
var Logger = require('./logger');
var files = require('./files');
var Processor = require('./processor');
var Schedule = require('./schedule');
var bar = require('./bar');
var log = Logger.create('nbob');

function timeSince(start) {
	var ms = Date.now() - start;
	return ms < 100 ? ms + 'ms' : (ms / 1000).toFixed(1) + 's';
}

var argCommands = args.commands;
if (argCommands.length === 0) {
	// TODO: Show brief help here, add option --help for showing full help
	log.help('No command(s) specified');
}

log.info('nBob v%s: %s v%s', pkg.version, config.project.name, config.project.version);
log.debug('Commands:', argCommands.map(function(command) {
	return commands.getName(command);
}).join(', '));
log.debug('Options:', args.options);
log.spam('Config:', config);

process.chdir(args.options.dir);

files.init();
log.spam('Files:\n  %s', files.getPaths().join('\n  '));

var schedule = new Schedule();
var readOnlyPromises = [];
_.filter(commands.select(argCommands), 'processor').forEach(function(command) {
	var processorName = command.processor;
	var commandName = commands.getName(command);
	var shortName = commands.getName(command, true);
	var pConfig = _.extend(config[commandName], { project: config.project });
	var pLog = Logger.create(shortName);
	var processorStart;

	var processor = require('./processor/' + processorName + '.js');
	if (processor instanceof Processor) {
		processor.init(pConfig, pLog);

		var batches = processor.getBatches(files.getPaths());

		var batchesLen = batches.length;
		var onFinishBatch = _.after(batchesLen, function onFinishBatches() {
			pLog.resume();
			pLog.info('Finished %d batches after: %s', batchesLen, timeSince(processorStart));
		});

		batches.forEach(function(batch, i) {
			var isReadOnly = batch.isReadOnly();

			bar.add(shortName);

			// TODO: Optimize by dividing across multiple cores using child_process.fork()
			// I'm thinking per batch, but sending file data back and forth to children might be inefficient.
			// Unfortunately, looking for shared memory modules I did not find anything good and cross platform.
			// Alternatively we could simply just fork read-only processors like jshint.
			schedule.addParallel(function() {
				if (!processorStart) {
					processorStart = Date.now();
					pLog.debug('Start %d batches', batchesLen);
					pLog.pause();
				}

				var inputFiles = files.getList(batch.inputPaths);
				var promise = processor.process(inputFiles, batch.outputPaths).then(function(outputFiles) {
					if (isReadOnly) {
						assert.strictEqual(outputFiles, undefined, 'Read only processor should return no output');
					} else {
						var outputPaths = _.pluck(outputFiles, 'path').sort();
						var batchOutputPaths = batch.outputPaths.slice().sort(); // slice to work around Object.freeze
						assert.deepEqual(outputPaths, batchOutputPaths, 'Processor output should match batch');

						files.process(inputFiles, outputFiles);
					}

					bar.finish(shortName);
					pLog.spam('Finished batch #%d after: %s', i + 1, timeSince(processorStart));
					onFinishBatch();
				}, function(error) {
					pLog.info('Rejected batch #%d after: %s', i + 1, timeSince(processorStart));
					onFinishBatch();
					return Promise.reject(error);
				});
				if (isReadOnly) {
					readOnlyPromises.push(promise);
					return Promise.resolve();
				}
				return promise;
			});
{'fraction_non_alphanumeric': 0.14953030498005404, 'fraction_numerical': 0.0028310384763865653, 'mean_word_length': 4.233670033670034, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1139183', 'n_tokens_mistral': 2612, 'n_tokens_neox': 2272, 'n_words': 643}