<commit_before>import * as sdk from 'botpress/sdk'
import _ from 'lodash'

import { IStanEngine } from 'src/backend/stan'
import pickSpellChecked from '../../election/spellcheck-handler'
import { mapPredictOutput } from '../../stan/api-mapper'
import { EventUnderstanding } from '../typings'

interface BotDefinition {
  botId: string
  defaultLanguage: string
}

type RawEventUnderstanding = Omit<EventUnderstanding, 'detectedLanguage'>

// TODO: rm this class and put all the logic inside Bot. This class no longer has a reason to be.
export class ScopedPredictionHandler {
  private defaultLanguage: string
  private _botId: string

  constructor(
    bot: BotDefinition,
    private _engine: IStanEngine,
    private _modelsByLang: _.Dictionary<string>,
    private _logger: sdk.Logger
  ) {
    this.defaultLanguage = bot.defaultLanguage
    this._botId = bot.botId
  }

  async predict(textInput: string, anticipatedLanguage: string): Promise<EventUnderstanding> {
    const { defaultLanguage } = this

    let detectedLanguage: string | undefined
    try {
      detectedLanguage = await this._engine.detectLanguage(this._botId, textInput, Object.values(this._modelsByLang))
    } catch (err) {
      let msg = `An error occured when detecting language for input "${textInput}"\n`
      msg += `Falling back on default language: ${defaultLanguage}.`
      this._logger.attachError(err).error(msg)
    }

    let nluResults: RawEventUnderstanding | undefined

    const isDefined = _.negate(_.isUndefined)
    const languagesToTry = _([detectedLanguage, anticipatedLanguage, defaultLanguage])
      .filter(isDefined)
      .uniq()
      .value()

    for (const lang of languagesToTry) {
      const res = await this.tryPredictInLanguage(textInput, lang)
      nluResults = res && { ...res }

      if (!this.isEmpty(nluResults) && !this.isError(nluResults)) {
        break
      }
    }

    if (this.isEmpty(nluResults)) {
      throw new Error(`No model found for the following languages: ${languagesToTry}`)
    }

    return { ...nluResults, detectedLanguage }
  }

  private async tryPredictInLanguage(textInput: string, language: string): Promise<RawEventUnderstanding | undefined> {
    if (!this._modelsByLang[language]) {
      return
    }

    try {
      const rawOriginalOutput = await this._engine.predict(this._botId, textInput, this._modelsByLang[language])
      const originalOutput = mapPredictOutput(rawOriginalOutput)
      const { spellChecked } = originalOutput
      return { ...originalOutput, spellChecked, errored: false, language }
    } catch (err) {
      const modelId = this._modelsByLang[language]<commit_msg>fix(nlu): better error logging when no model found. (#5541)<commit_after>import * as sdk from 'botpress/sdk'
import _ from 'lodash'

import { IStanEngine } from 'src/backend/stan'
import pickSpellChecked from '../../election/spellcheck-handler'
import { mapPredictOutput } from '../../stan/api-mapper'
import { EventUnderstanding } from '../typings'

interface BotDefinition {
  botId: string
  defaultLanguage: string
}

type RawEventUnderstanding = Omit<EventUnderstanding, 'detectedLanguage'>

// TODO: rm this class and put all the logic inside Bot. This class no longer has a reason to be.
export class ScopedPredictionHandler {
  private defaultLanguage: string
  private _botId: string

  constructor(
    bot: BotDefinition,
    private _engine: IStanEngine,
    private _modelsByLang: _.Dictionary<string>,
    private _logger: sdk.Logger
  ) {
    this.defaultLanguage = bot.defaultLanguage
    this._botId = bot.botId
  }

  async predict(textInput: string, anticipatedLanguage: string): Promise<EventUnderstanding> {
    const { defaultLanguage } = this

    let detectedLanguage: string | undefined
    try {
      detectedLanguage = await this._engine.detectLanguage(this._botId, textInput, Object.values(this._modelsByLang))
    } catch (err) {
      let msg = `An error occured when detecting language for input "${textInput}"\n`
      msg += `Falling back on default language: ${defaultLanguage}.`
      this._logger.attachError(err).error(msg)
    }

    let nluResults: RawEventUnderstanding | undefined

    const isDefined = _.negate(_.isUndefined)
    const languagesToTry = _([detectedLanguage, anticipatedLanguage, defaultLanguage])
      .filter(isDefined)
      .uniq()
      .value()

    for (const lang of languagesToTry) {
      const res = await this.tryPredictInLanguage(textInput, lang)
      nluResults = res && { ...res }

      if (!this.isEmpty(nluResults) && !this.isError(nluResults)) {
        break
      }
    }

    if (this.isEmpty(nluResults)) {
      throw new Error(
        `No model found for the following languages: [${languagesToTry.join(', ')}]. Please train your chatbot.`
      )
    }

    return { ...nluResults, detectedLanguage }
  }

  private async tryPredictInLanguage(textInput: string, language: string): Promise<RawEventUnderstanding | undefined> {
    if (!this._modelsByLang[language]) {
      return
    }

    try {
      const rawOriginalOutput = await this._engine.predict(this._botId, textInput, this._modelsByLang[language])
      const originalOutput = mapPredictOutput(rawOriginalOutput)
      const { spellChecked } = originalOutput
      return { ...originalOutput, spellChecked, errored: false, language }
    } catch (err) {
      const modelId = this._modelsByLang[language]
{'fraction_non_alphanumeric': 0.1026827012025902, 'fraction_numerical': 0.0007400555041628122, 'mean_word_length': 3.5891341256366722, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3435565', 'n_tokens_mistral': 1536, 'n_tokens_neox': 1509, 'n_words': 414}