<commit_before>import { Logger, MLToolkit, NLU } from 'botpress/sdk'
import _ from 'lodash'

import { isPatternValid } from '../tools/patterns-utils'
import { Engine2, ListEntity, Tools, TrainingSession } from '../typings'

import CRFExtractor2 from './crf-extractor2'
import { computeModelHash, Model } from './model-service'
import { Predict, PredictInput, Predictors, PredictOutput } from './predict-pipeline'
import { computeKmeans, ProcessIntents, Trainer, TrainInput, TrainOutput } from './training-pipeline'

export default class E2 implements Engine2 {
  private static tools: Tools
  private predictorsByLang: _.Dictionary<Predictors> = {}
  private modelsByLang: _.Dictionary<Model> = {}

  constructor(private defaultLanguage: string, private botId: string, private logger: Logger) {}

  static provideTools(tools: Tools) {
    E2.tools = tools
  }

  async train(
    intentDefs: NLU.IntentDefinition[],
    entityDefs: NLU.EntityDefinition[],
    languageCode: string,
    trainingSession: TrainingSession
  ): Promise<Model> {
    this.logger.info(`Started ${languageCode} training for bot ${this.botId}`)

    const list_entities = entityDefs
      .filter(ent => ent.type === 'list')
      .map(e => {
        return {
          name: e.name,
          fuzzyTolerance: e.fuzzy,
          sensitive: e.sensitive,
          synonyms: _.chain(e.occurences)
            .keyBy('name')
            .mapValues('synonyms')
            .value()
        } as ListEntity
      })

    const pattern_entities = entityDefs
      .filter(ent => ent.type === 'pattern' && isPatternValid(ent.pattern))
      .map(ent => ({
        name: ent.name,
        pattern: ent.pattern,
        examples: [], // TODO add this to entityDef
        matchCase: ent.matchCase,
        sensitive: ent.sensitive
      }))

    const contexts = _.chain(intentDefs)
      .flatMap(i => i.contexts)
      .uniq()
      .value()

    const input: TrainInput = {
      botId: this.botId,
      trainingSession,
      languageCode,
      list_entities,
      pattern_entities,
      contexts,
      intents: intentDefs
        .filter(x => !!x.utterances[languageCode])
        .map(x => ({
          name: x.name,
          contexts: x.contexts,
          utterances: x.utterances[languageCode],
          slot_definitions: x.slots
        }))
    }

    // Model should be build here, Trainer should not have any idea of how this is stored
    // Error handling should be done here
    const model = await Trainer(input, E2.tools)
    model.hash = computeModelHash(intentDefs, entityDefs)
    if (model.success) {
      E2.tools.reportTrainingProgress(this.botId, 'Training complete', {
        ...trainingSession,
        progress: 1,
        status: 'done'
      })
      this.logger.info(`Successfully finished ${languageCode} training for bot: ${this.botId}`)
    }

    return model
  }

  private modelAlreadyLoaded(model: Model) {
    return (
      this.predictorsByLang[model.languageCode] !== undefined &&
      this.modelsByLang[model.languageCode] !== undefined &&
      this.modelsByLang[model.languageCode].hash === model.hash
    )<commit_msg>removed hash compare, needs a migration<commit_after>import { Logger, MLToolkit, NLU } from 'botpress/sdk'
import _ from 'lodash'

import { isPatternValid } from '../tools/patterns-utils'
import { Engine2, ListEntity, Tools, TrainingSession } from '../typings'

import CRFExtractor2 from './crf-extractor2'
import { computeModelHash, Model } from './model-service'
import { Predict, PredictInput, Predictors, PredictOutput } from './predict-pipeline'
import { computeKmeans, ProcessIntents, Trainer, TrainInput, TrainOutput } from './training-pipeline'

export default class E2 implements Engine2 {
  private static tools: Tools
  private predictorsByLang: _.Dictionary<Predictors> = {}
  private modelsByLang: _.Dictionary<Model> = {}

  constructor(private defaultLanguage: string, private botId: string, private logger: Logger) {}

  static provideTools(tools: Tools) {
    E2.tools = tools
  }

  async train(
    intentDefs: NLU.IntentDefinition[],
    entityDefs: NLU.EntityDefinition[],
    languageCode: string,
    trainingSession: TrainingSession
  ): Promise<Model> {
    this.logger.info(`Started ${languageCode} training for bot ${this.botId}`)

    const list_entities = entityDefs
      .filter(ent => ent.type === 'list')
      .map(e => {
        return {
          name: e.name,
          fuzzyTolerance: e.fuzzy,
          sensitive: e.sensitive,
          synonyms: _.chain(e.occurences)
            .keyBy('name')
            .mapValues('synonyms')
            .value()
        } as ListEntity
      })

    const pattern_entities = entityDefs
      .filter(ent => ent.type === 'pattern' && isPatternValid(ent.pattern))
      .map(ent => ({
        name: ent.name,
        pattern: ent.pattern,
        examples: [], // TODO add this to entityDef
        matchCase: ent.matchCase,
        sensitive: ent.sensitive
      }))

    const contexts = _.chain(intentDefs)
      .flatMap(i => i.contexts)
      .uniq()
      .value()

    const input: TrainInput = {
      botId: this.botId,
      trainingSession,
      languageCode,
      list_entities,
      pattern_entities,
      contexts,
      intents: intentDefs
        .filter(x => !!x.utterances[languageCode])
        .map(x => ({
          name: x.name,
          contexts: x.contexts,
          utterances: x.utterances[languageCode],
          slot_definitions: x.slots
        }))
    }

    // Model should be build here, Trainer should not have any idea of how this is stored
    // Error handling should be done here
    const model = await Trainer(input, E2.tools)
    model.hash = computeModelHash(intentDefs, entityDefs)
    if (model.success) {
      E2.tools.reportTrainingProgress(this.botId, 'Training complete', {
        ...trainingSession,
        progress: 1,
        status: 'done'
      })
      this.logger.info(`Successfully finished ${languageCode} training for bot: ${this.botId}`)
    }

    return model
  }

  private modelAlreadyLoaded(model: Model) {
    return (
      this.predictorsByLang[model.languageCode] !== undefined &&
      this.modelsByLang[model.languageCode] !== undefined &&
      _.isEqual(this.modelsByLang[model.languageCode].data.input, model.data.input)
      // TODO compare hash instead (need a migration)
      // this.modelsByLang[model.languageCode].hash === model.hash
    )
{'fraction_non_alphanumeric': 0.11745539177657098, 'fraction_numerical': 0.0027928626842513575, 'mean_word_length': 2.743321718931475, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2261676', 'n_tokens_mistral': 1903, 'n_tokens_neox': 1860, 'n_words': 460}