<commit_before>#!/usr/bin/env node

var
	_        = require('lodash'),
	Analyzer = require('../index').Analyzer,
	assert   = require('assert'),
	bole     = require('bole'),
	path     = require('path'),
	redis    = require('redis'),
	argv     = require('yargs')
		.option('l', {
			alias: 'listen',
			default: '0.0.0.0:3337',
			description: 'host:port pair to listen on'
		})
		.option('silent', {
			description: 'silence analyzer-specific logging',
			type: 'boolean',
			default: false
		})
		.version(function() { return require('../package').version; })
		.describe('version', 'show version information')
		.alias('h', 'help')
		.help('help')
		.usage('Usage: $0 --listen localhost:3333 config.json')
		.showHelpOnFail(true)
		.demand(1)
		.argv
	;

// set up logging
if (!argv.silent)
{
	if (process.env.NODE_ENV === 'dev')
	{
		var prettystream = require('bistre')({time: true});
		prettystream.pipe(process.stdout);
		bole.output({ level: 'debug', stream: prettystream });
	}
	else
		bole.output({level: 'info', stream: process.stdout});
}
var logger = bole('config');

var config = require(path.resolve(process.cwd(), argv._[0]));
assert(config.rules, 'You must provide a rule set in the `rules` section of your config.');
if (process.env.REDIS_URL || process.env.REDIS)
	config.redis = redis.createClient(process.env.REDIS_URL || process.env.REDIS);

var pair = argv.listen.split(':');
var runoptions = {
	listen: {
		host: pair[0],
		port: pair[1]
	},
	rules: [],
	outputs: []
};

// Now we attempt to load the requested modules & yell if we can't.
_.each(config.rules, function(config, name)
{
	var item = construct('rules', name, config);
	if (item)
	{
		runoptions.rules.push(item);
		logger.info('configured rule ' + name);
	}
	else
		logger.warn('could not load rule ' + name);
});

_.each(config.outputs, function(config, name)
{
	var item = construct('outputs', name, config);
	if (item)
	{
		runoptions.outputs.push(item);
		logger.info('configured output ' + name);
	}
	else
		logger.warn('could not load output ' + name);
});

function construct(type, name, config)
{
	var Item;

	try { Item = require('../lib/' + type + '/' + name); }<commit_msg>Pass redis down to every rule & output.<commit_after>#!/usr/bin/env node

var
	_        = require('lodash'),
	Analyzer = require('../index').Analyzer,
	assert   = require('assert'),
	bole     = require('bole'),
	path     = require('path'),
	redis    = require('redis'),
	argv     = require('yargs')
		.option('l', {
			alias: 'listen',
			default: '0.0.0.0:3337',
			description: 'host:port pair to listen on'
		})
		.option('silent', {
			description: 'silence analyzer-specific logging',
			type: 'boolean',
			default: false
		})
		.version(function() { return require('../package').version; })
		.describe('version', 'show version information')
		.alias('h', 'help')
		.help('help')
		.usage('Usage: $0 --listen localhost:3333 config.json')
		.showHelpOnFail(true)
		.demand(1)
		.argv
	;

// set up logging
if (!argv.silent)
{
	if (process.env.NODE_ENV === 'dev')
	{
		var prettystream = require('bistre')({time: true});
		prettystream.pipe(process.stdout);
		bole.output({ level: 'debug', stream: prettystream });
	}
	else
		bole.output({level: 'info', stream: process.stdout});
}
var logger = bole('config');

var config = require(path.resolve(process.cwd(), argv._[0]));
assert(config.rules, 'You must provide a rule set in the `rules` section of your config.');
if (process.env.REDIS_URL || process.env.REDIS)
	config.redis = redis.createClient(process.env.REDIS_URL || process.env.REDIS);

var pair = argv.listen.split(':');
var runoptions = {
	listen: {
		host: pair[0],
		port: pair[1]
	},
	rules: [],
	outputs: []
};

// Now we attempt to load the requested modules & yell if we can't.
_.each(config.rules, function(ruleConfig, name)
{
	ruleConfig.redis = config.redis;
	var item = construct('rules', name, ruleConfig);
	if (item)
	{
		runoptions.rules.push(item);
		logger.info('configured rule ' + name);
	}
	else
		logger.warn('could not load rule ' + name);
});

_.each(config.outputs, function(config, name)
{
	ruleConfig.redis = config.redis;
	var item = construct('outputs', name, ruleConfig);
	if (item)
	{
		runoptions.outputs.push(item);
		logger.info('configured output ' + name);
	}
	else
		logger.warn('could not load output ' + name);
});

function construct(type, name, config)
{
	var Item;

	try { Item = require('../lib/' + type + '/' + name); }
{'fraction_non_alphanumeric': 0.18361836183618363, 'fraction_numerical': 0.00765076507650765, 'mean_word_length': 3.88998899889989, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1952468', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1541, 'n_words': 368}