<commit_before>var bunyan = require('bunyan'),
	_ = require('lodash'),
	fs = require('fs'),
	wrench = require('wrench'),
	path = require('path');

/**
 * create a restify server logger and setup not only the main
 * logger but also the request logger.
 */
function createRestifyLogger (server, options) {
	var ConsoleLogger = require('./console');

	var logs = options && options.logs || './logs';
	if (!fs.existsSync(logs)) {
		wrench.mkdirSyncRecursive(logs);
	}

	var consoleLogger = new ConsoleLogger(options),
		config = {
			name: server.name || 'server',
			serializers: {
				req: bunyan.stdSerializers.req,
				res: bunyan.stdSerializers.res
			},
			streams: [
				{
					level: options && options.level || 'info',
					type:'raw',
					stream: consoleLogger
				}
			]
		},
		serverLogger = bunyan.createLogger(config),
		logDir = logs,
		requestLogger = bunyan.createLogger({
			name: server.name || 'requests',
			serializers: {
				req: bunyan.stdSerializers.req,
				res: bunyan.stdSerializers.res
			},
			streams: [
				{
					level: options && options.level || 'info',
					path: path.join(logDir,'requests.log')
				}
			]
		});

	// for each request, create a request logger
	server.pre(function(req, resp, next){
		// we prefix with date to make it easier to sort logs by timestamp
		var name = 'request-'+req.getId(),
			logname = path.join(logDir, name + '.log'),
			log = bunyan.createLogger({
				name: name,
				serializers: {
					req: bunyan.stdSerializers.req,
					res: bunyan.stdSerializers.res
				},
				streams: [
					{
						level: options && options.level || 'trace',
						path: logname
					},
					{
						level: options && options.level || 'info',
						type:'raw',
						stream: consoleLogger
					}
				]
			});
		log.info({req_id:req.getId(),req:req,res:resp,start:true,ignore:true},'start');
		req.log = log;
		req.started = process.hrtime();
		req.log.logname = logname;
		req.log.name = name;
		next();
	});

	// by default, listen for the server's 'after' event but also let the
	// creator tell us to use a different event. this is nice when you have
	// additional things you want to do before ending the logging (after the server might
	// have sent the response) that you want to log or capture before ending
	var afterEvent = options && options.afterEvent||'after';

	server.on(afterEvent,function(req, res){
		// see if we've already calculated the duration and if so, use it
		var duration = req.duration;
		if (!duration) {
			// otherwise we need to calculated
			var time = process.hrtime(req.started);
			duration = (time[0] / 1000) + (time[1]*1.0e-6);
			req.duration = duration;
		}
		req.log.info({req_id:req.getId(), body: req.body, ignore:true});
		requestLogger.info({res:res, req_id: req.getId(), req:req, name: req.log.name, logname: req.log.logname, duration: duration});
	});

	server.log = serverLogger;

	return serverLogger;
}

/**
 * create a default logger
 */
function createDefaultLogger(options) {
	var ConsoleLogger = require('./console');
	var config = _.merge({
		name:'logger',
		streams: [
			{
				level: options && options.level || 'trace',
				type: 'raw',
				stream: new ConsoleLogger(options)
			}
		]
	}, options, function(a, b) {
		return _.isArray(a) ? a.concat(b) : undefined;<commit_msg>write out metadata as well as the logfile<commit_after>var bunyan = require('bunyan'),
	_ = require('lodash'),
	fs = require('fs'),
	wrench = require('wrench'),
	path = require('path');

/**
 * create a restify server logger and setup not only the main
 * logger but also the request logger.
 */
function createRestifyLogger (server, options) {
	var ConsoleLogger = require('./console');

	var logs = options && options.logs || './logs';
	if (!fs.existsSync(logs)) {
		wrench.mkdirSyncRecursive(logs);
	}

	var consoleLogger = new ConsoleLogger(options),
		config = {
			name: server.name || 'server',
			serializers: {
				req: bunyan.stdSerializers.req,
				res: bunyan.stdSerializers.res
			},
			streams: [
				{
					level: options && options.level || 'info',
					type:'raw',
					stream: consoleLogger
				}
			]
		},
		serverLogger = bunyan.createLogger(config),
		logDir = logs,
		requestLogger = bunyan.createLogger({
			name: server.name || 'requests',
			serializers: {
				req: bunyan.stdSerializers.req,
				res: bunyan.stdSerializers.res
			},
			streams: [
				{
					level: options && options.level || 'info',
					path: path.join(logDir,'requests.log')
				}
			]
		});

	// for each request, create a request logger
	server.pre(function(req, resp, next){
		// we prefix with date to make it easier to sort logs by timestamp
		var name = 'request-'+req.getId(),
			logname = path.join(logDir, name + '.log'),
			log = bunyan.createLogger({
				name: name,
				serializers: {
					req: bunyan.stdSerializers.req,
					res: bunyan.stdSerializers.res
				},
				streams: [
					{
						level: options && options.level || 'trace',
						path: logname
					},
					{
						level: options && options.level || 'info',
						type:'raw',
						stream: consoleLogger
					}
				]
			});
		log.info({req_id:req.getId(),req:req,res:resp,start:true,ignore:true},'start');
		req.log = log;
		req.started = process.hrtime();
		req.log.logname = logname;
		req.log.name = name;
		next();
	});

	// by default, listen for the server's 'after' event but also let the
	// creator tell us to use a different event. this is nice when you have
	// additional things you want to do before ending the logging (after the server might
	// have sent the response) that you want to log or capture before ending
	var afterEvent = options && options.afterEvent||'after';

	server.on(afterEvent,function(req, res){
		// see if we've already calculated the duration and if so, use it
		var duration = req.duration;
		if (!duration) {
			// otherwise we need to calculated
			var time = process.hrtime(req.started);
			duration = (time[0] / 1000) + (time[1]*1.0e-6);
			req.duration = duration;
		}
		req.log.info({req_id:req.getId(), body: req.body, ignore:true});
		var result = {res:res, req_id: req.getId(), req:req, name: req.log.name, logname: req.log.logname, duration: duration};
		requestLogger.info(result);
		fs.writeFile(req.log.logname+'.metadata',JSON.stringify(result));
	});

	server.log = serverLogger;

	return serverLogger;
}

/**
 * create a default logger
 */
function createDefaultLogger(options) {
	var ConsoleLogger = require('./console');
	var config = _.merge({
		name:'logger',
		streams: [
			{
				level: options && options.level || 'trace',
				type: 'raw',
				stream: new ConsoleLogger(options)
			}
		]
	}, options, function(a, b) {
		return _.isArray(a) ? a.concat(b) : undefined;
{'fraction_non_alphanumeric': 0.14642910615361582, 'fraction_numerical': 0.0026950142236861806, 'mean_word_length': 3.265644955300128, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1944552', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2144, 'n_words': 580}