<commit_before>import { Logger } from 'botpress/sdk'
import { extractArchive } from 'core/misc/archive'
import { GhostService } from 'core/services'
import { BotService } from 'core/services/bot-service'
import { CMSService } from 'core/services/cms'
import { Router } from 'express'
import _ from 'lodash'
import mkdirp from 'mkdirp'
import path from 'path'
import tmp from 'tmp'

import { CustomRouter } from '../customRouter'

export class VersioningRouter extends CustomRouter {
  constructor(
    logger: Logger,
    private ghost: GhostService,
    private botService: BotService,
    private cmsService: CMSService
  ) {
    super('Versioning', logger, Router({ mergeParams: true }))
    this.setupRoutes()
  }

  setupRoutes() {
    this.router.get(
      '/pending',
      this.asyncMiddleware(async (req, res) => {
        const botIds = await this.botService.getBotsIds()
        res.send(await this.ghost.getPending(botIds))
      })
    )

    this.router.get(
      '/export',
      this.asyncMiddleware(async (req, res) => {
        const botIds = await this.botService.getBotsIds()
        const tarball = await this.ghost.exportArchive(botIds)

        res.writeHead(200, {
          'Content-Type': 'application/tar+gzip',
          'Content-Disposition': `attachment; filename=archive_${Date.now()}.tgz`,
          'Content-Length': tarball.length
        })
        res.end(tarball)
      })
    )

    this.router.post(
      '/changes',
      this.asyncMiddleware(async (req, res) => {
        const tmpDir = tmp.dirSync({ unsafeCleanup: true })

        try {
          await this.extractArchiveFromRequest(req, tmpDir.name)

          res.send({ changes: await this.ghost.listFileChanges(tmpDir.name) })
        } catch (error) {
          res.status(500).send('Error while pushing changes')
        } finally {
          tmpDir.removeCallback()
        }
      })
    )

    // Force update of the production files by the local files
    this.router.post(
      '/update',
      this.asyncMiddleware(async (req, res) => {
        const tmpDir = tmp.dirSync({ unsafeCleanup: true })

        try {
          await this.extractArchiveFromRequest(req, tmpDir.name)
          await this.ghost.forceUpdate(tmpDir.name)

          const botsIds = await this.botService.getBotsIds()
          await Promise.map(botsIds, id => this.cmsService.broadcastInvalidateForBot(id))

          res.sendStatus(200)
        } catch (error) {
          res.status(500).send('Error while pushing changes')
        } finally {
          tmpDir.removeCallback()
        }
      })
    )
  }

  extractArchiveFromRequest = async (request, folder) => {
    const dataFolder = path.join(folder, 'data')
    await mkdirp.sync(dataFolder)

    const buffer: Buffer[] = []
    request.on('data', chunk => buffer.push(chunk))

    await Promise.fromCallback(cb => request.on('end', cb))
    await extractArchive(Buffer.concat(buffer), dataFolder)
  }
}<commit_msg>mount & unmount bots after push/pull<commit_after>import { Logger } from 'botpress/sdk'
import { extractArchive } from 'core/misc/archive'
import { GhostService } from 'core/services'
import { BotService } from 'core/services/bot-service'
import { CMSService } from 'core/services/cms'
import { Router } from 'express'
import _ from 'lodash'
import mkdirp from 'mkdirp'
import path from 'path'
import tmp from 'tmp'

import { CustomRouter } from '../customRouter'

export class VersioningRouter extends CustomRouter {
  constructor(
    logger: Logger,
    private ghost: GhostService,
    private botService: BotService,
    private cmsService: CMSService
  ) {
    super('Versioning', logger, Router({ mergeParams: true }))
    this.setupRoutes()
  }

  setupRoutes() {
    this.router.get(
      '/pending',
      this.asyncMiddleware(async (req, res) => {
        const botIds = await this.botService.getBotsIds()
        res.send(await this.ghost.getPending(botIds))
      })
    )

    this.router.get(
      '/export',
      this.asyncMiddleware(async (req, res) => {
        const botIds = await this.botService.getBotsIds()
        const tarball = await this.ghost.exportArchive(botIds)

        res.writeHead(200, {
          'Content-Type': 'application/tar+gzip',
          'Content-Disposition': `attachment; filename=archive_${Date.now()}.tgz`,
          'Content-Length': tarball.length
        })
        res.end(tarball)
      })
    )

    this.router.post(
      '/changes',
      this.asyncMiddleware(async (req, res) => {
        const tmpDir = tmp.dirSync({ unsafeCleanup: true })

        try {
          await this.extractArchiveFromRequest(req, tmpDir.name)

          res.send({ changes: await this.ghost.listFileChanges(tmpDir.name) })
        } catch (error) {
          res.status(500).send('Error while pushing changes')
        } finally {
          tmpDir.removeCallback()
        }
      })
    )

    // Force update of the production files by the local files
    this.router.post(
      '/update',
      this.asyncMiddleware(async (req, res) => {
        const tmpDir = tmp.dirSync({ unsafeCleanup: true })
        const beforeBotIds = await this.botService.getBotsIds()

        try {
          await this.extractArchiveFromRequest(req, tmpDir.name)
          await this.ghost.forceUpdate(tmpDir.name)

          // we let the caches invalidate and propagate TODO: do this properly instead of waiting
          await Promise.delay(5000)

          // Unmount all previous bots and re-mount only the remaining (and new) bots
          await Promise.map(beforeBotIds, id => this.botService.unmountBot(id))
          const afterBotIds = await this.botService.getBotsIds()
          await Promise.map(afterBotIds, id => this.botService.mountBot(id))

          res.sendStatus(200)
        } catch (error) {
          res.status(500).send('Error while pushing changes')
        } finally {
          tmpDir.removeCallback()
        }
      })
    )
  }

  extractArchiveFromRequest = async (request, folder) => {
    const dataFolder = path.join(folder, 'data')
    await mkdirp.sync(dataFolder)

    const buffer: Buffer[] = []
    request.on('data', chunk => buffer.push(chunk))

    await Promise.fromCallback(cb => request.on('end', cb))
    await extractArchive(Buffer.concat(buffer), dataFolder)
  }
}
{'fraction_non_alphanumeric': 0.12037478164205177, 'fraction_numerical': 0.004446561854851517, 'mean_word_length': 2.663757998836533, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7045149', 'n_tokens_mistral': 1895, 'n_tokens_neox': 1849, 'n_words': 424}