<commit_before>#!/usr/local/bin/node

'use strict'

const 
  paths = {
    HttpServer: './lib/HttpServer',
    middleware: './lib/middleware',
    config: './config',
    log: 'startup.log',
    logsys: 'sys.log'
  }
,
  config = require(paths.config)
,
  fs = require('fs'),
  path = require('path'),
  os = require('os')
,
  { logLevel, logsPath } = config,
  Log = require('logerr')({ logLevel, logsPath }),
  log = new Log(paths.log, { rotate: true }),
  logsys = new Log(paths.logsys, { rotate: true })
,
  HttpServer = require(paths.HttpServer)
,
  noop = () => {}
;

const middlewareLoader = new Promise( (resolve, reject) => {
  fs.readdir(path.resolve(__dirname, paths.middleware), (err, res) => {
    if (err) { reject(err) }
    // filter out dot files (like vim .swp files)
    resolve(res.filter( file => file[0] !== '.' ).map( fn => require(`${paths.middleware}/${fn}`) ))
  }) 
})

const { 
  name, 
  port, 
  api,
  onError = noop, 
  onRequest = noop, 
  socketTimeout = 120000
} = config

// set root
process.env.ROOT = config.root
// set logs path
process.env.LOGS_PATH = logsPath

middlewareLoader.then( middleware => {
  log.log('...startup')
  const server = new HttpServer({ 
    name,
    port, 
    api,
    middleware
  })
  server.timeout = socketTimeout
  server.on('error', onError)
  server.on('request', onRequest)
  /** Every hour: 
   * 1. log the number of current connections
   * 2. log the freemem/totalmem
   * 3. log the system activity via the load average calculation
   * 4. log the system uptime
   * 5. log the process uptime
   * 6. log the cpu usage
   * 7. log the memory usage
  **/
  setInterval(() => {
    server.getConnections( (err, count) => {
      if (err) log.error(err)
      logsys.log(`${count} connections open`)
      logsys.info(`${os.freemem()}/${os.totalmem()} memory available`)
      logsys.info(`${os.loadavg()}:::<< load average "should be less than number of logical CPUs in the system" (${os.cpus().length})`)
      logsys.info(`${os.uptime()}:::<< system uptime`)
      logsys.info(`${process.uptime()}:::<< process uptime`)
      const cpuUsage = process.cpuUsage()
      logsys.info(`${cpuUsage.user}:::<< user cpu usage`)
      logsys.info(`${cpuUsage.system}:::<< system cpu usage`)
      logsys.info(`${process.memoryUsage()}:::<< process memory usage`)
    }) 
  }, 3600000)
  // handle Ctrl+c
  process.on(os.constants.signals.SIGINT, () => {
    server.close(() => {
      console.log(`${server.name || 'server'} shut-down gracefully - `)           
      process.exit(1)
    })
  })
}).catch( err => {
  log.error(`STARTUP ERROR ${err}`)
  throw err 
})
<commit_msg>set NODE_PATH so webapp can benefit from awning's node_modules<commit_after>#!/usr/local/bin/node

'use strict'

const 
  paths = {
    HttpServer: './lib/HttpServer',
    middleware: './lib/middleware',
    config: './config',
    log: 'startup.log',
    logsys: 'sys.log'
  }
,
  config = require(paths.config)
,
  fs = require('fs'),
  path = require('path'),
  os = require('os')
,
  { logLevel, logsPath } = config,
  Log = require('logerr')({ logLevel, logsPath }),
  log = new Log(paths.log, { rotate: true }),
  logsys = new Log(paths.logsys, { rotate: true })
,
  HttpServer = require(paths.HttpServer)
,
  noop = () => {}
;

const middlewareLoader = new Promise( (resolve, reject) => {
  fs.readdir(path.resolve(__dirname, paths.middleware), (err, res) => {
    if (err) { reject(err) }
    // filter out dot files (like vim .swp files)
    resolve(res.filter( file => file[0] !== '.' ).map( fn => require(`${paths.middleware}/${fn}`) ))
  }) 
})

const { 
  name, 
  port, 
  api,
  onError = noop, 
  onRequest = noop, 
  socketTimeout = 120000
} = config

// set root
process.env.ROOT = config.root
// set logs path
process.env.LOGS_PATH = logsPath
// set NODE_PATH
process.env.NODE_PATH = path.join(__dirname, 'node_modules')

middlewareLoader.then( middleware => {
  log.log('...startup')
  const server = new HttpServer({ 
    name,
    port, 
    api,
    middleware
  })
  server.timeout = socketTimeout
  server.on('error', onError)
  server.on('request', onRequest)
  /** Every hour: 
   * 1. log the number of current connections
   * 2. log the freemem/totalmem
   * 3. log the system activity via the load average calculation
   * 4. log the system uptime
   * 5. log the process uptime
   * 6. log the cpu usage
   * 7. log the memory usage
  **/
  setInterval(() => {
    server.getConnections( (err, count) => {
      if (err) log.error(err)
      logsys.log(`${count} connections open`)
      logsys.info(`${os.freemem()}/${os.totalmem()} memory available`)
      logsys.info(`${os.loadavg()}:::<< load average "should be less than number of logical CPUs in the system" (${os.cpus().length})`)
      logsys.info(`${os.uptime()}:::<< system uptime`)
      logsys.info(`${process.uptime()}:::<< process uptime`)
      const cpuUsage = process.cpuUsage()
      logsys.info(`${cpuUsage.user}:::<< user cpu usage`)
      logsys.info(`${cpuUsage.system}:::<< system cpu usage`)
      logsys.info(`${process.memoryUsage()}:::<< process memory usage`)
    }) 
  }, 3600000)
  // handle Ctrl+c
  process.on(os.constants.signals.SIGINT, () => {
    server.close(() => {
      console.log(`${server.name || 'server'} shut-down gracefully - `)           
      process.exit(1)
    })
  })
}).catch( err => {
  log.error(`STARTUP ERROR ${err}`)
  throw err 
})

{'fraction_non_alphanumeric': 0.17886778535865758, 'fraction_numerical': 0.008113590263691683, 'mean_word_length': 3.087415222305953, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2687560', 'n_tokens_mistral': 1908, 'n_tokens_neox': 1849, 'n_words': 445}