<commit_before>// eslint-disable-next-line import/order
import rewire from '../../sdk/rewire'
// eslint-disable-next-line import/order

import bytes from 'bytes'
import chalk from 'chalk'
import cluster from 'cluster'
import { LogLevel } from 'core/logger/enums'
import { centerText } from 'core/logger/utils'
import { copyDir } from 'core/misc/pkg-fs'
import _ from 'lodash'
import * as NLUEngine from 'nlu/engine'
import path from 'path'
import { setupMasterNode, WORKER_TYPES } from '../../cluster'

import Logger from '../../simple-logger'
import API, { APIOptions } from './api'

global.rewire = rewire as any
const debug = DEBUG('api')

const GH_TYPINGS_FILE = 'https://github.com/botpress/botpress/blob/master/src/bp/nlu-server/typings_v1.d.ts'
const GH_TRAIN_INPUT_EXAMPLE = 'https://github.com/botpress/botpress/blob/master/src/bp/nlu-server/train-example.json'

type ArgV = APIOptions & {
  languageURL: string
  languageAuthToken?: string
  ducklingURL: string
  ducklingEnabled: boolean
}

const makeEngine = async (options: ArgV, logger: Logger) => {
  const loggerWrapper: NLUEngine.Logger = {
    debug: (msg: string) => logger.debug(msg),
    info: (msg: string) => logger.info(msg),
    warning: (msg: string, err?: Error) => (err ? logger.attachError(err).warn(msg) : logger.warn(msg)),
    error: (msg: string, err?: Error) => (err ? logger.attachError(err).error(msg) : logger.error(msg))
  }

  try {
    const { ducklingEnabled, ducklingURL, modelCacheSize, languageURL, languageAuthToken } = options
    const config: NLUEngine.Config = {
      languageSources: [
        {
          endpoint: languageURL,
          authToken: languageAuthToken
        }
      ],
      ducklingEnabled,
      ducklingURL,
      modelCacheSize,
      legacyElection: false
    }

    const engine = await NLUEngine.makeEngine(config, loggerWrapper)
    return engine
  } catch (err) {
    // TODO: Make lang provider throw if it can't connect.
    logger
      .attachError(err)
      .error(
        'There was an error while initializing Engine tools. Check out the connection to your language and Duckling server.'
      )
    process.exit(1)
  }
}

export default async function(options: ArgV) {
  const logger = new Logger('Launcher')
  if (cluster.isMaster) {
    setupMasterNode(logger)
    return
  } else if (cluster.isWorker && process.env.WORKER_TYPE !== WORKER_TYPES.WEB) {
    return
  }

  for (const dir of ['./pre-trained', './stop-words']) {
    await copyDir(path.resolve(__dirname, '../nlu/engine/assets', dir), path.resolve(process.APP_DATA_PATH, dir))
  }

  if (!bytes(options.bodySize)) {
    throw new Error(`Specified body-size "${options.bodySize}" has an invalid format.`)
  }

  options.modelDir = options.modelDir || path.join(process.APP_DATA_PATH, 'models')<commit_msg>fix(nlu): update pre-trained models path (#4819)<commit_after>// eslint-disable-next-line import/order
import rewire from '../../sdk/rewire'
// eslint-disable-next-line import/order

import bytes from 'bytes'
import chalk from 'chalk'
import cluster from 'cluster'
import { LogLevel } from 'core/logger/enums'
import { centerText } from 'core/logger/utils'
import { copyDir } from 'core/misc/pkg-fs'
import _ from 'lodash'
import * as NLUEngine from 'nlu/engine'
import path from 'path'
import { setupMasterNode, WORKER_TYPES } from '../../cluster'

import Logger from '../../simple-logger'
import API, { APIOptions } from './api'

global.rewire = rewire as any
const debug = DEBUG('api')

const GH_TYPINGS_FILE = 'https://github.com/botpress/botpress/blob/master/src/bp/nlu/stan/typings_v1.d.ts'
const GH_TRAIN_INPUT_EXAMPLE = 'https://github.com/botpress/botpress/blob/master/src/bp/nlu/stan/train-example.json'

type ArgV = APIOptions & {
  languageURL: string
  languageAuthToken?: string
  ducklingURL: string
  ducklingEnabled: boolean
}

const makeEngine = async (options: ArgV, logger: Logger) => {
  const loggerWrapper: NLUEngine.Logger = {
    debug: (msg: string) => logger.debug(msg),
    info: (msg: string) => logger.info(msg),
    warning: (msg: string, err?: Error) => (err ? logger.attachError(err).warn(msg) : logger.warn(msg)),
    error: (msg: string, err?: Error) => (err ? logger.attachError(err).error(msg) : logger.error(msg))
  }

  try {
    const { ducklingEnabled, ducklingURL, modelCacheSize, languageURL, languageAuthToken } = options
    const config: NLUEngine.Config = {
      languageSources: [
        {
          endpoint: languageURL,
          authToken: languageAuthToken
        }
      ],
      ducklingEnabled,
      ducklingURL,
      modelCacheSize,
      legacyElection: false
    }

    const engine = await NLUEngine.makeEngine(config, loggerWrapper)
    return engine
  } catch (err) {
    // TODO: Make lang provider throw if it can't connect.
    logger
      .attachError(err)
      .error(
        'There was an error while initializing Engine tools. Check out the connection to your language and Duckling server.'
      )
    process.exit(1)
  }
}

export default async function(options: ArgV) {
  const logger = new Logger('Launcher')
  if (cluster.isMaster) {
    setupMasterNode(logger)
    return
  } else if (cluster.isWorker && process.env.WORKER_TYPE !== WORKER_TYPES.WEB) {
    return
  }

  for (const dir of ['./pre-trained', './stop-words']) {
    await copyDir(path.resolve(__dirname, '../engine/assets', dir), path.resolve(process.APP_DATA_PATH, dir))
  }

  if (!bytes(options.bodySize)) {
    throw new Error(`Specified body-size "${options.bodySize}" has an invalid format.`)
  }

  options.modelDir = options.modelDir || path.join(process.APP_DATA_PATH, 'models')
{'fraction_non_alphanumeric': 0.13153696153164332, 'fraction_numerical': 0.0014181882644921113, 'mean_word_length': 3.5208333333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6310903', 'n_tokens_mistral': 1795, 'n_tokens_neox': 1788, 'n_words': 452}