<commit_before>import * as sdk from 'botpress/sdk'
import crypto from 'crypto'
import _ from 'lodash'

import { TrainArtefacts, TrainInput, TrainOutput } from './training-pipeline'

export interface Model {
  hash: string
  languageCode: string
  startedAt: Date
  finishedAt: Date
  success: boolean
  data: {
    input: TrainInput
    output?: TrainOutput
    artefacts?: TrainArtefacts
  }
}

const MODELS_DIR = './models'
const MAX_MODELS_TO_KEEP = 2

function makeFileName(hash: string, lang: string): string {
  return `${hash}.${lang}.model`
}

// we might want to make this language specific
export function computeModelHash(intents: any, entities: any): string {
  return crypto
    .createHash('md5')
    .update(JSON.stringify({ intents, entities }))
    .digest('hex')
}

function serializeModel(model: Model): string {
  return JSON.stringify(_.omit(model, ['data.output', 'data.input.trainingSession']))
}

function deserializeModel(str: string): Model {
  const model = JSON.parse(str) as Model
  model.data.artefacts.slots_model = Buffer.from(model.data.artefacts.slots_model)
  return model
}

async function pruneModels(ghost: sdk.ScopedGhostService, languageCode: string): Promise<void | void[]> {
  const models = await listModelsForLang(ghost, languageCode)
  if (models.length > MAX_MODELS_TO_KEEP) {
    return Promise.map(models.slice(MAX_MODELS_TO_KEEP - 1), file => ghost.deleteFile(MODELS_DIR, file))
  }
}

async function listModelsForLang(ghost: sdk.ScopedGhostService, languageCode: string): Promise<string[]> {
  const endingPattern = makeFileName('*', languageCode)
  return await ghost.directoryListing(MODELS_DIR, endingPattern, undefined, undefined {orderBy: 'modifiedOn', order: 'desc'})
}

export async function getModel(ghost: sdk.ScopedGhostService, hash: string, lang: string): Promise<Model | void> {
  const fname = makeFileName(hash, lang)
  if (await ghost.fileExists(MODELS_DIR, fname)) {
    const strMod = await ghost.readFileAsString(MODELS_DIR, fname)
    return deserializeModel(strMod)
  }
}

export async function getLatestModel(ghost: sdk.ScopedGhostService, lang: string): Promise<Model | void> {
  const availableModels = await listModelsForLang(ghost, lang)
  if (availableModels.length === 0) {<commit_msg>feat(nlu):prune models on save<commit_after>import * as sdk from 'botpress/sdk'
import crypto from 'crypto'
import _ from 'lodash'

import { TrainArtefacts, TrainInput, TrainOutput } from './training-pipeline'

export interface Model {
  hash: string
  languageCode: string
  startedAt: Date
  finishedAt: Date
  success: boolean
  data: {
    input: TrainInput
    output?: TrainOutput
    artefacts?: TrainArtefacts
  }
}

const MODELS_DIR = './models'
const MAX_MODELS_TO_KEEP = 2

function makeFileName(hash: string, lang: string): string {
  return `${hash}.${lang}.model`
}

// we might want to make this language specific
export function computeModelHash(intents: any, entities: any): string {
  return crypto
    .createHash('md5')
    .update(JSON.stringify({ intents, entities }))
    .digest('hex')
}

function serializeModel(model: Model): string {
  return JSON.stringify(_.omit(model, ['data.output', 'data.input.trainingSession']))
}

function deserializeModel(str: string): Model {
  const model = JSON.parse(str) as Model
  model.data.artefacts.slots_model = Buffer.from(model.data.artefacts.slots_model)
  return model
}

async function pruneModels(ghost: sdk.ScopedGhostService, languageCode: string): Promise<void | void[]> {
  const models = await listModelsForLang(ghost, languageCode)
  if (models.length > MAX_MODELS_TO_KEEP) {
    return Promise.map(models.slice(MAX_MODELS_TO_KEEP), file => ghost.deleteFile(MODELS_DIR, file))
  }
}

async function listModelsForLang(ghost: sdk.ScopedGhostService, languageCode: string): Promise<string[]> {
  const endingPattern = makeFileName('*', languageCode)
  return await ghost.directoryListing(MODELS_DIR, endingPattern, undefined, undefined, {
    sortOrder: { column: 'modifiedOn', desc: true }
  })
}

export async function getModel(ghost: sdk.ScopedGhostService, hash: string, lang: string): Promise<Model | void> {
  const fname = makeFileName(hash, lang)
  if (await ghost.fileExists(MODELS_DIR, fname)) {
    const strMod = await ghost.readFileAsString(MODELS_DIR, fname)
    return deserializeModel(strMod)
  }
}

export async function getLatestModel(ghost: sdk.ScopedGhostService, lang: string): Promise<Model | void> {
  const availableModels = await listModelsForLang(ghost, lang)
  if (availableModels.length === 0) {
{'fraction_non_alphanumeric': 0.11449879359508665, 'fraction_numerical': 0.0015354244351831541, 'mean_word_length': 5.2465753424657535, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3856661', 'n_tokens_mistral': 1442, 'n_tokens_neox': 1432, 'n_words': 365}