<commit_before>import * as sdk from 'botpress/sdk'

import { listModelsForLang, Model, MODELS_DIR, pruneModels, saveModel } from '../backend/model-service'

const migration: sdk.ModuleMigration = {
  info: {
    description: 'Prune old models and compress old models',
    target: 'bot',
    type: 'config'
  },
  up: async ({ bp, metadata }: sdk.ModuleMigrationOpts): Promise<sdk.MigrationResult> => {
    const migrateModels = async (bot: sdk.BotConfig) => {
      const ghost = bp.ghost.forBot(bot.id)

      return Promise.mapSeries(bot.languages, async lang => {
        await pruneModels(ghost, lang)
        const modNames = await listModelsForLang(ghost, lang)

        return Promise.map(modNames, async mod => {
          try {
            const model: Model = await ghost.readFileAsObject(MODELS_DIR, mod)
            // Triggers model compression
            return saveModel(ghost, model, model.hash)
          } catch (err) {
            return
          }
        })
      })
    }

    if (!metadata.botId) {
      const bots = await bp.bots.getAllBots()
      await Promise.map(bots.values(), migrateModels)
    }

    return { success: true, message: 'Model compression completed successfully' }
  }
}<commit_msg>remove model if not up to date<commit_after>import * as sdk from 'botpress/sdk'

import { listModelsForLang, Model, MODELS_DIR, pruneModels, saveModel } from '../backend/model-service'

const migration: sdk.ModuleMigration = {
  info: {
    description: 'Prune old models and compress old models',
    target: 'bot',
    type: 'config'
  },
  up: async ({ bp, metadata }: sdk.ModuleMigrationOpts): Promise<sdk.MigrationResult> => {
    const migrateModels = async (bot: sdk.BotConfig) => {
      const ghost = bp.ghost.forBot(bot.id)

      return Promise.mapSeries(bot.languages, async lang => {
        await pruneModels(ghost, lang)
        const modNames = await listModelsForLang(ghost, lang)

        return Promise.map(modNames, async mod => {
          try {
            const model: Model = await ghost.readFileAsObject(MODELS_DIR, mod)
            if (!model.hash) {
              return ghost.deleteFile(MODELS_DIR, mod) // model is really outdated
            }
            return saveModel(ghost, model, model.hash) // Triggers model compression
          } catch (err) {
            return
          }
        })
      })
    }

    if (!metadata.botId) {
      const bots = await bp.bots.getAllBots()
      await Promise.map(bots.values(), migrateModels)
    }

    return { success: true, message: 'Model compression completed successfully' }
  }
}
{'fraction_non_alphanumeric': 0.11175337186897881, 'fraction_numerical': 0.0, 'mean_word_length': 2.5708390646492436, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6543257', 'n_tokens_mistral': 776, 'n_tokens_neox': 769, 'n_words': 201}