<commit_before>// Copyright (c) 2015, <NAME> (twitter.com/evanxsummers)
// ISC license, see http://github.com/evanx/redexutil/LICENSE

import assert from 'assert';
import bunyan from 'bunyan';
import lodash from 'lodash';

const defaultLevel = 'info';
const levels = ['debug', 'info', 'warn', 'error'];
const extraLevels = ['state', 'digest', 'child'];
const allLevels = levels.concat(extraLevels);
const digestLimit = 100;
const state = {
   limit: 10,
   stats: {},
   logging: {
      error: [],
      warn: [],
      info: [],
      debug: [],
      digest: []
   }
};

function getStats(name) {
   let stats = state.stats[name];
   if (!stats) {
      stats = {
         counts: {},
         averages: {},
         peaks: {},
      };
      state.stats[name] = stats;
   }
   return stats;
}

function increment(name, prop) {
   let stats = getStats(name);
   let value = stats.counters[prop] || 0;
   value += 1;
   stats.counters[prop] = value;
   return value;
}

function peak(name, prop, value) {
   let count = increment(name, prop);
   let stats = getStats(name);
   let average = stats.averages[prop];
   average = ((count - 1) * average + value)/count;
   stats.averages[prop] = average;
   if (!stats.peaks.hasOwnProperty(prop)) {
      stats.peaks[prop] = value;
      return value;
   }
   let peak = stats.peaks[prop];
   if (value > peak) {
      stats.peaks[prop] = value;
      peak = value;
   }
   return peak;
}

function basename(file) {
   var matcher = file.match(/([^\/]+)\.[a-z]+$/);
   if (matcher) {
      return matcher[1];
   } else {
      return file;
   }
}

module.exports = {
   pub() {
      return Object.assign({}, state.stats, state.logging);
   },
   counters() {
      return state.counters;
   },
   create(name, level) {
      name = basename(name);
      level = level || global.loggerLevel || process.env.loggerLevel || defaultLevel;
      if (lodash.includes(levels, level)) {
         let logger = bunyan.createLogger({name, level});
         return decorate(logger, name, level);
      } else {
         assert(lodash.includes(extraLevels, level), 'level: ' + level);
         return decorate(null, name, level);
      }
   }
};

function logging(logger, name, loggerLevel, context, level, args, count) {
   increment(name, level);
   args = [].slice.call(args); // convert arguments to array
   if (!lodash.isEmpty(context)) {
      if (lodash.isArray(context)) {
         args = context.concat(args);
      }
   }
   if (!state.logging.hasOwnProperty(level)) {
      args.splice(0, 0, 'Invalid level: ' + level);
      level = 'warn';
   }
   if (logger) {
      if (level === 'digest') {
         if (count < digestLimit/10 && count % digestLimit === 0) {
            logger.info('digest', count, ...args);<commit_msg>logger time for average stats<commit_after>// Copyright (c) 2015, <NAME> (twitter.com/evanxsummers)
// ISC license, see http://github.com/evanx/redexutil/LICENSE

import assert from 'assert';
import bunyan from 'bunyan';
import lodash from 'lodash';

const defaultLevel = 'info';
const levels = ['debug', 'info', 'warn', 'error'];
const extraLevels = ['state', 'digest', 'child'];
const allLevels = levels.concat(extraLevels);
const digestLimit = 100;
const state = {
   limit: 10,
   stats: {},
   logging: {
      error: [],
      warn: [],
      info: [],
      debug: [],
      digest: []
   }
};

function getStats(name) {
   let stats = state.stats[name];
   if (!stats) {
      stats = {
         counts: {},
         averages: {},
         peaks: {},
      };
      state.stats[name] = stats;
   }
   return stats;
}

function increment(name, prop) {
   let stats = getStats(name);
   let value = stats.counts[prop] || 0;
   value += 1;
   stats.counts[prop] = value;
   return value;
}

function peak(name, prop, value) {
   let count = increment(name, prop);
   let stats = getStats(name);
   let average = stats.averages[prop];
   average = ((count - 1) * average + value)/count;
   stats.averages[prop] = average;
   if (!stats.peaks.hasOwnProperty(prop)) {
      stats.peaks[prop] = value;
      return value;
   }
   let peak = stats.peaks[prop];
   if (value > peak) {
      stats.peaks[prop] = value;
      peak = value;
   }
   return peak;
}

function basename(file) {
   var matcher = file.match(/([^\/]+)\.[a-z]+$/);
   if (matcher) {
      return matcher[1];
   } else {
      return file;
   }
}

module.exports = {
   pub() {
      return Object.assign({}, state.stats, state.logging);
   },
   counters() { // TODO deprecate
      return {};
   },
   create(name, level) {
      name = basename(name);
      level = level || global.loggerLevel || process.env.loggerLevel || defaultLevel;
      if (lodash.includes(levels, level)) {
         let logger = bunyan.createLogger({name, level});
         return decorate(logger, name, level);
      } else {
         assert(lodash.includes(extraLevels, level), 'level: ' + level);
         return decorate(null, name, level);
      }
   }
};

function logging(logger, name, loggerLevel, context, level, args, count) {
   increment(name, level);
   args = [].slice.call(args); // convert arguments to array
   if (!lodash.isEmpty(context)) {
      if (lodash.isArray(context)) {
         args = context.concat(args);
      }
   }
   if (!state.logging.hasOwnProperty(level)) {
      args.splice(0, 0, 'Invalid level: ' + level);
      level = 'warn';
   }
   if (logger) {
      if (level === 'digest') {
         if (count < digestLimit/10 && count % digestLimit === 0) {
            logger.info('digest', count, ...args);
{'fraction_non_alphanumeric': 0.16379000359582885, 'fraction_numerical': 0.006472491909385114, 'mean_word_length': 2.6335728282168516, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1908023', 'n_tokens_mistral': 1856, 'n_tokens_neox': 1806, 'n_words': 440}