<commit_before>const TEST_ENV = process.env.NODE_ENV === 'test'
const Discord = require('discord.js')
const COLORS = {
  Error: '\x1b[31m',
  Success: '\x1b[32m',
  Warning: '\x1b[33m',
  Debug: '\x1b[33m',
  reset: '\x1b[0m'
}
const CONSTRUCTORS = [Discord.Guild, Discord.TextChannel, Discord.Role, Discord.User]
const LOG_DATES = require('../config.js').log.dates === true
const PREFIXES = ['G', 'C', 'R', 'U']
const TYPES = ['Command', 'Guild', 'Cycle', 'INIT', 'General', 'Debug', 'Owner', 'Web']
const LEVELS = ['Error', 'Success', 'Warning', 'Info']
const MAXLEN = TYPES.reduce((a, b) => a.length > b.length ? a : b).length + LEVELS.reduce((a, b) => a.length > b.length ? a : b).length + 1 // Calculate uniform spacing
let suppressedLevels = []
let showTraceByDefault = false

function formatConsoleDate (date) {
  // http://stackoverflow.com/questions/18814221/adding-timestamps-to-all-console-messages
  const hour = date.getHours()
  const minutes = date.getMinutes()
  const seconds = date.getSeconds()
  const milliseconds = date.getMilliseconds()
  return `[${((hour < 10) ? '0' + hour : hour)}:${((minutes < 10) ? '0' + minutes : minutes)}:${((seconds < 10) ? '0' + seconds : seconds)}.${('00' + milliseconds).slice(-3)}] `
}

class _Logger {
  constructor (type) {
    this.type = type
    LEVELS.forEach(level => {
      this[level.toLowerCase()] = this._log(level)
    })
  }

  _parseDetails (details) {
    if (details.length === 0) return { identifier: '' }
    let error
    let det = ''
    for (var q = 0; q < details.length; ++q) {
      const item = details[q]
      if (!item) continue
      const i = CONSTRUCTORS.indexOf(item.constructor)
      if (i === -1) {
        if (item instanceof Error) error = item
        continue
      }
      const pre = PREFIXES[i]
      det += item.id && (item.name || item.username) ? `(${pre}: ${item.id}, ${item.name || item.username}) ` : item.id ? `(${pre} ${item.id}) ` : item.name ? `(${pre} ${item.name}) ` : ''
    }
    return { identifier: det, err: error, printStack: error && details[details.length - 1] === true }
  }

  _log (level) {
    let intro = `${this.type} ${level}`
    for (let i = intro.length; i < MAXLEN; ++i) intro += ' '
    const color = COLORS[level] ? COLORS[level] : ''
    const reset = COLORS.reset ? COLORS.reset : ''
    return (contents, ...details) => {
      if (TEST_ENV) return
      if (suppressedLevels.includes(level.toLowerCase())) return
      const extra = this._parseDetails(details)
      console.log(`${LOG_DATES ? formatConsoleDate(new Date()) : ''}${color}${intro}${reset} | ${extra.identifier}${contents}${extra.err ? ` (${extra.err}${extra.err.code ? `, Code ${extra.err.code}` : ''})` : ''}`)
      if (extra.err && (extra.printStack || showTraceByDefault)) console.log(extra.err.stack) // Print stack trace
    }
  }
}

TYPES.forEach(type => {
  exports[type.toLowerCase()] = new _Logger(type)
})

exports.suppressLevel = level => {
  if (Array.isArray(level)) suppressedLevels = suppressedLevels.concat(level)
  else suppressedLevels.push(level)
}
<commit_msg>Add DEV MODE to log when applicable<commit_after>const TEST_ENV = process.env.NODE_ENV === 'test'
const config = require('../config.js')
const Discord = require('discord.js')
const COLORS = {
  Error: '\x1b[31m',
  Success: '\x1b[32m',
  Warning: '\x1b[33m',
  Debug: '\x1b[33m',
  reset: '\x1b[0m'
}
const CONSTRUCTORS = [Discord.Guild, Discord.TextChannel, Discord.Role, Discord.User]
const LOG_DATES = require('../config.js').log.dates === true
const PREFIXES = ['G', 'C', 'R', 'U']
const TYPES = ['Command', 'Guild', 'Cycle', 'INIT', 'General', 'Debug', 'Owner', 'Web']
const LEVELS = ['Error', 'Success', 'Warning', 'Info']
const MAXLEN = TYPES.reduce((a, b) => a.length > b.length ? a : b).length + LEVELS.reduce((a, b) => a.length > b.length ? a : b).length + 1 // Calculate uniform spacing
let suppressedLevels = []
let showTraceByDefault = false

function formatConsoleDate (date) {
  // http://stackoverflow.com/questions/18814221/adding-timestamps-to-all-console-messages
  const hour = date.getHours()
  const minutes = date.getMinutes()
  const seconds = date.getSeconds()
  const milliseconds = date.getMilliseconds()
  return `[${((hour < 10) ? '0' + hour : hour)}:${((minutes < 10) ? '0' + minutes : minutes)}:${((seconds < 10) ? '0' + seconds : seconds)}.${('00' + milliseconds).slice(-3)}] `
}

class _Logger {
  constructor (type) {
    this.type = type
    LEVELS.forEach(level => {
      this[level.toLowerCase()] = this._log(level)
    })
  }

  _parseDetails (details) {
    if (details.length === 0) return { identifier: '' }
    let error
    let det = ''
    for (var q = 0; q < details.length; ++q) {
      const item = details[q]
      if (!item) continue
      const i = CONSTRUCTORS.indexOf(item.constructor)
      if (i === -1) {
        if (item instanceof Error) error = item
        continue
      }
      const pre = PREFIXES[i]
      det += item.id && (item.name || item.username) ? `(${pre}: ${item.id}, ${item.name || item.username}) ` : item.id ? `(${pre} ${item.id}) ` : item.name ? `(${pre} ${item.name}) ` : ''
    }
    return { identifier: det, err: error, printStack: error && details[details.length - 1] === true }
  }

  _log (level) {
    let intro = `${this.type} ${level}`
    for (let i = intro.length; i < MAXLEN; ++i) intro += ' '
    const color = COLORS[level] ? COLORS[level] : ''
    const reset = COLORS.reset ? COLORS.reset : ''
    return (contents, ...details) => {
      if (TEST_ENV) return
      if (suppressedLevels.includes(level.toLowerCase())) return
      const extra = this._parseDetails(details)
      const dev = config.dev === true ? `${COLORS.Error}[DEV MODE]${reset}` : ''
      const date = LOG_DATES ? formatConsoleDate(new Date()) : ''
      const extraErr = extra.err ? ` (${extra.err}${extra.err.code ? `, Code ${extra.err.code}` : ''})` : ''
      console.log(`${dev}${date}${color}${intro}${reset} | ${extra.identifier}${contents}${extraErr}`)
      if (extra.err && (extra.printStack || showTraceByDefault)) console.log(extra.err.stack) // Print stack trace
    }
  }
}

TYPES.forEach(type => {
  exports[type.toLowerCase()] = new _Logger(type)
})

exports.suppressLevel = level => {
  if (Array.isArray(level)) suppressedLevels = suppressedLevels.concat(level)
  else suppressedLevels.push(level)
}

{'fraction_non_alphanumeric': 0.1927066960075448, 'fraction_numerical': 0.012260295504558314, 'mean_word_length': 3.5127659574468084, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '188848', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2183, 'n_words': 500}