<commit_before>'use strict'

// //////////////////////////////////////////////////////////////////////////
// actionhero framework in node.js
// http://www.actionherojs.com
// https://github.com/actionhero/actionhero

const path = require('path')
const async = require('async')
const glob = require('glob')

// HELPERS ///

const fatalError = function (api, errors, type) {
  if (errors && !(errors instanceof Array)) { errors = [errors] }
  if (errors) {
    if (api.log) {
      api.log(`Error with initializer step: ${type}`, 'emerg')
      errors.forEach((error) => { api.log(error.stack, 'emerg') })
    } else {
      console.error('Error with initializer step: ' + type)
      errors.forEach((error) => { console.error(error.stack) })
    }
    api.commands.stop.call(api, () => {
      process.exit(1)
    })
  }
}

const sortNumber = function (a, b) {
  return a - b
}

let startCount = 0

const flattenOrderedInitialzer = function (collection) {
  let output = []
  let keys = []
  for (let key in collection) {
    keys.push(parseInt(key))
  }
  keys.sort(sortNumber)
  keys.forEach((key) => {
    collection[key].forEach((d) => {
      output.push(d)
    })
  })

  return output
}

// ACTIONHERO //

const actionhero = function () {
  this.initializers = {}
  this.api = {
    running: false,
    initialized: false,
    shuttingDown: false
  }
}

actionhero.prototype.initialize = function (params, callback) {
  this.api.commands = {
    initialize: (params, callback) => { this.initialize(params, callback) },
    start: (params, callback) => { this.start(params, callback) },
    stop: (callback) => { this.stop(callback) },
    restart: (callback) => { this.restart(callback) }
  }

  this.api.projectRoot = process.cwd()

  if (process.env.project_root) {
    this.api.projectRoot = process.env.project_root
  } else if (process.env.projectRoot) {
    this.api.projectRoot = process.env.projectRoot
  } else if (process.env.PROJECT_ROOT) {
    this.api.projectRoot = process.env.PROJECT_ROOT
  }

  if (!callback && typeof params === 'function') {
    callback = params; params = {}
  }
  if (params === null) { params = {} }
  this.startingParams = params
  this.api._startingParams = this.startingParams

  this.api.initializerDefaults = {
    load: 1000,
    start: 1000,
    stop: 1000
  }

  let loadInitializerRankings = {}
  let startInitializerRankings = {}
  let stopInitializerRankings = {}

  this.configInitializers = []
  this.loadInitializers = []
  this.startInitializers = []
  this.stopInitializers = [];

  // we need to load the config first
  [
    path.resolve(__dirname, 'initializers', 'utils.js'),
    path.resolve(__dirname, 'initializers', 'config.js')
  ].forEach((file) => {
    let filename = file.replace(/^.*[\\/]/, '')
    let initializer = filename.split('.')[0]
    delete require.cache[require.resolve(file)]
    this.initializers[initializer] = require(file)
    this.configInitializers.push((next) => {
      this.initializers[initializer].initialize(this.api, next)
    })
  })

  this.configInitializers.push(() => {
    let customInitializers = []
    let duplicatedInitializers = []
    this.api.config.general.paths.initializer.forEach((startPath) => {
      customInitializers = customInitializers.concat(glob.sync(path.join(startPath, '**', '*.js'), {follow: true}))
    })
    // load all other initializers
    this.api.utils.arrayUniqueify(
      glob.sync(path.join(__dirname, 'initializers', '**', '*.js'), {follow: true})
      .sort()
      .concat(
        customInitializers
        .sort()
      )
    ).forEach((f) => {
      let file = path.normalize(f)
      let initializer = path.basename(f).split('.')[0]
      let fileParts = file.split('.')
      let ext = fileParts[(fileParts.length - 1)]
      if (ext === 'js') {
        // check if initializer already exists (exclude utils and config)
        if (this.initializers[initializer] &&
           file !== path.resolve(__dirname, 'initializers', 'utils.js') &&
           file !== path.resolve(__dirname, 'initializers', 'config.js')) {
          duplicatedInitializers.push(file)
        } else {
          delete require.cache[require.resolve(file)]
          this.initializers[initializer] = require(file)
        }

        const loadFunction = (next) => {
          this.api.watchFileAndAct(file, () => {
            this.api.log(`*** Rebooting due to initializer change (${file}) ***`, 'info')
            this.api.commands.restart()
          })

          if (typeof this.initializers[initializer].initialize === 'function') {
            if (typeof this.api.log === 'function') { this.api.log(`Loading initializer: ${initializer}`, 'debug', file) }
            try {
              this.initializers[initializer].initialize(this.api, (error) => {
                try { this.api.log(`Loaded initializer: ${initializer}`, 'debug', file) } catch (e) { }
                next(error)
              })
            } catch (e) {
              this.api.log(`Exception occured in initializer: ${initializer}  during load`, 'debug', e)
              next(e)
            }
          } else {
            next()
          }
        }

        const startFunction = (next) => {
          if (typeof this.initializers[initializer].start === 'function') {
            if (typeof this.api.log === 'function') { this.api.log(`Starting initializer: ${initializer}`, 'debug', file) }
            try {
              this.initializers[initializer].start(this.api, (error) => {
                this.api.log(`Started initializer: ${initializer}`, 'debug', file)
                next(error)
              })
            } catch (e) {
              this.api.log(`Exception occured in initializer: ${initializer} during start`, 'debug', e)
              next(e)
            }
          } else {
            next()
          }
        }

        const stopFunction = (next) => {
          if (typeof this.initializers[initializer].stop === 'function') {
            if (typeof this.api.log === 'function') { this.api.log(`Stopping initializer: ${initializer}`, 'debug', file) }
            try {
              this.initializers[initializer].stop(this.api, (error) => {
                this.api.log(`Stopped initializer: ${initializer}`, 'debug', file)
                next(error)
              })
            } catch (e) {
              this.api.log(`Exception occured in initializer: ${initializer} during stop`, 'debug', e)
              next(e)
            }
          } else {
            next()
          }
        }

        if (this.initializers[initializer].loadPriority === undefined) {
          this.initializers[initializer].loadPriority = this.api.initializerDefaults.load
        }<commit_msg>warning log on init errors<commit_after>'use strict'

// //////////////////////////////////////////////////////////////////////////
// actionhero framework in node.js
// http://www.actionherojs.com
// https://github.com/actionhero/actionhero

const path = require('path')
const async = require('async')
const glob = require('glob')

// HELPERS ///

const fatalError = function (api, errors, type) {
  if (errors && !(errors instanceof Array)) { errors = [errors] }
  if (errors) {
    if (api.log) {
      api.log(`Error with initializer step: ${type}`, 'emerg')
      errors.forEach((error) => { api.log(error.stack, 'emerg') })
    } else {
      console.error('Error with initializer step: ' + type)
      errors.forEach((error) => { console.error(error.stack) })
    }
    api.commands.stop.call(api, () => {
      process.exit(1)
    })
  }
}

const sortNumber = function (a, b) {
  return a - b
}

let startCount = 0

const flattenOrderedInitialzer = function (collection) {
  let output = []
  let keys = []
  for (let key in collection) {
    keys.push(parseInt(key))
  }
  keys.sort(sortNumber)
  keys.forEach((key) => {
    collection[key].forEach((d) => {
      output.push(d)
    })
  })

  return output
}

// ACTIONHERO //

const actionhero = function () {
  this.initializers = {}
  this.api = {
    running: false,
    initialized: false,
    shuttingDown: false
  }
}

actionhero.prototype.initialize = function (params, callback) {
  this.api.commands = {
    initialize: (params, callback) => { this.initialize(params, callback) },
    start: (params, callback) => { this.start(params, callback) },
    stop: (callback) => { this.stop(callback) },
    restart: (callback) => { this.restart(callback) }
  }

  this.api.projectRoot = process.cwd()

  if (process.env.project_root) {
    this.api.projectRoot = process.env.project_root
  } else if (process.env.projectRoot) {
    this.api.projectRoot = process.env.projectRoot
  } else if (process.env.PROJECT_ROOT) {
    this.api.projectRoot = process.env.PROJECT_ROOT
  }

  if (!callback && typeof params === 'function') {
    callback = params; params = {}
  }
  if (params === null) { params = {} }
  this.startingParams = params
  this.api._startingParams = this.startingParams

  this.api.initializerDefaults = {
    load: 1000,
    start: 1000,
    stop: 1000
  }

  let loadInitializerRankings = {}
  let startInitializerRankings = {}
  let stopInitializerRankings = {}

  this.configInitializers = []
  this.loadInitializers = []
  this.startInitializers = []
  this.stopInitializers = [];

  // we need to load the config first
  [
    path.resolve(__dirname, 'initializers', 'utils.js'),
    path.resolve(__dirname, 'initializers', 'config.js')
  ].forEach((file) => {
    let filename = file.replace(/^.*[\\/]/, '')
    let initializer = filename.split('.')[0]
    delete require.cache[require.resolve(file)]
    this.initializers[initializer] = require(file)
    this.configInitializers.push((next) => {
      this.initializers[initializer].initialize(this.api, next)
    })
  })

  this.configInitializers.push(() => {
    let customInitializers = []
    let duplicatedInitializers = []
    this.api.config.general.paths.initializer.forEach((startPath) => {
      customInitializers = customInitializers.concat(glob.sync(path.join(startPath, '**', '*.js'), {follow: true}))
    })
    // load all other initializers
    this.api.utils.arrayUniqueify(
      glob.sync(path.join(__dirname, 'initializers', '**', '*.js'), {follow: true})
      .sort()
      .concat(
        customInitializers
        .sort()
      )
    ).forEach((f) => {
      let file = path.normalize(f)
      let initializer = path.basename(f).split('.')[0]
      let fileParts = file.split('.')
      let ext = fileParts[(fileParts.length - 1)]
      if (ext === 'js') {
        // check if initializer already exists (exclude utils and config)
        if (this.initializers[initializer] &&
           file !== path.resolve(__dirname, 'initializers', 'utils.js') &&
           file !== path.resolve(__dirname, 'initializers', 'config.js')) {
          duplicatedInitializers.push(file)
        } else {
          delete require.cache[require.resolve(file)]
          this.initializers[initializer] = require(file)
        }

        const loadFunction = (next) => {
          this.api.watchFileAndAct(file, () => {
            this.api.log(`*** Rebooting due to initializer change (${file}) ***`, 'info')
            this.api.commands.restart()
          })

          if (typeof this.initializers[initializer].initialize === 'function') {
            if (typeof this.api.log === 'function') { this.api.log(`Loading initializer: ${initializer}`, 'debug', file) }
            try {
              this.initializers[initializer].initialize(this.api, (error) => {
                try { this.api.log(`Loaded initializer: ${initializer}`, 'debug', file) } catch (e) { }
                next(error)
              })
            } catch (e) {
              this.api.log(`Exception occured in initializer: ${initializer}  during load`, 'warning', e)
              next(e)
            }
          } else {
            next()
          }
        }

        const startFunction = (next) => {
          if (typeof this.initializers[initializer].start === 'function') {
            if (typeof this.api.log === 'function') { this.api.log(`Starting initializer: ${initializer}`, 'debug', file) }
            try {
              this.initializers[initializer].start(this.api, (error) => {
                this.api.log(`Started initializer: ${initializer}`, 'debug', file)
                next(error)
              })
            } catch (e) {
              this.api.log(`Exception occured in initializer: ${initializer} during start`, 'warning', e)
              next(e)
            }
          } else {
            next()
          }
        }

        const stopFunction = (next) => {
          if (typeof this.initializers[initializer].stop === 'function') {
            if (typeof this.api.log === 'function') { this.api.log(`Stopping initializer: ${initializer}`, 'debug', file) }
            try {
              this.initializers[initializer].stop(this.api, (error) => {
                this.api.log(`Stopped initializer: ${initializer}`, 'debug', file)
                next(error)
              })
            } catch (e) {
              this.api.log(`Exception occured in initializer: ${initializer} during stop`, 'warning', e)
              next(e)
            }
          } else {
            next()
          }
        }

        if (this.initializers[initializer].loadPriority === undefined) {
          this.initializers[initializer].loadPriority = this.api.initializerDefaults.load
        }
{'fraction_non_alphanumeric': 0.16388084094792363, 'fraction_numerical': 0.0025258153183270187, 'mean_word_length': 2.3214902541327413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5644560', 'n_tokens_mistral': 3997, 'n_tokens_neox': 3917, 'n_words': 796}