<commit_before>import axios from 'axios'
import * as sdk from 'botpress/sdk'
import { Paging } from 'botpress/sdk'
import _ from 'lodash'
import nanoid from 'nanoid/generate'

import { QnaEntry, QnaItem } from './qna'

export const NLU_PREFIX = '__qna__'
const DEFAULT_CATEGORY = 'global'

const safeId = (length = 10) => nanoid('1234567890abcdefghijklmnopqrsuvwxyz', length)

const slugify = s => (s || '').toLowerCase().replace(/[^a-z0-9]/g, '_')

const getIntentId = id => `${NLU_PREFIX}${id}`

const makeID = (qna: QnaEntry) => {
  const firstQuestion = qna.questions[Object.keys(qna.questions)[0]][0]
  return `${safeId()}_${slugify(firstQuestion)
    .replace(/^_+/, '')
    .substring(0, 50)
    .replace(/_+$/, '')}`
}

const normalizeQuestions = questions =>
  questions
    .map(q =>
      q
        .replace(/[\r\n]+/g, ' ')
        .replace(/\s+/g, ' ')
        .trim()
    )
    .filter(Boolean)

export default class Storage {
  private bp: typeof sdk
  private config
  private botId: string
  private categories: string[]

  constructor(bp: typeof sdk, config, botId) {
    this.bp = bp
    this.config = config
    this.botId = botId

    if (config.qnaCategories && config.qnaCategories.length > 0) {
      this.categories = config.qnaCategories
        .split(',')
        .map(x => x.trim())
        .filter(x => x.length)
    }
  }

  private async getAxiosConfig() {
    return this.bp.http.getAxiosConfigForBot(this.botId, { localUrl: true })
  }

  async initialize() {
    await this.migrate11To12()
    await this.syncQnaToNlu()
  }

  // @Deprecated > 12
  async migrate11To12() {
    const bot = await this.bp.bots.getBotById(this.botId)
    const qnas = await this.fetchQNAs()

    return await Promise.all(
      qnas.map(({ id, data }) => {
        const initial = _.cloneDeep(data)
        const questions = data.questions
        const answers = data.answers
        if (!data.category) {
          data.category = 'global'
        }
        if (_.isArray(questions)) {
          data.questions = {
            [bot.defaultLanguage]: questions
          }
        }
        if (_.isArray(answers)) {
          data.answers = {
            [bot.defaultLanguage]: answers
          }
        }

        if (!_.isEqual(initial, data)) {
          return this.update(data, id)
        }
      })
    )
  }

  // TODO Find better way to implement. When manually copying QNA, intents are not created.
  // Manual edit & save of each one is required for the intent to be created.
  async syncQnaToNlu() {
    const axiosConfig = await this.getAxiosConfig()
    const allQuestions = await this.fetchQNAs()
    const { data: allIntents } = await axios.get(`/mod/nlu/intents`, axiosConfig)

    const qnaItemsToSync = allQuestions.filter(
      qnaItem => qnaItem.data.enabled && !_.find(allIntents, i => i.name === getIntentId(qnaItem.id).toLowerCase())
    )

    return await Promise.mapSeries(qnaItemsToSync, async item => {
      await this.checkForDuplicatedQuestions(item.data)
      await this.createNLUIntentFromQnaItem(item)
    })
  }

  private async createNLUIntentFromQnaItem(qnaItem: QnaItem): Promise<void> {
    const axiosConfig = await this.getAxiosConfig()
    const utterances = {}
    for (const lang in qnaItem.data.questions) {<commit_msg>fix(qna): redundant question already exists check<commit_after>import axios from 'axios'
import * as sdk from 'botpress/sdk'
import { Paging } from 'botpress/sdk'
import _ from 'lodash'
import nanoid from 'nanoid/generate'

import { QnaEntry, QnaItem } from './qna'

export const NLU_PREFIX = '__qna__'
const DEFAULT_CATEGORY = 'global'

const safeId = (length = 10) => nanoid('1234567890abcdefghijklmnopqrsuvwxyz', length)

const slugify = s => (s || '').toLowerCase().replace(/[^a-z0-9]/g, '_')

const getIntentId = id => `${NLU_PREFIX}${id}`

const makeID = (qna: QnaEntry) => {
  const firstQuestion = qna.questions[Object.keys(qna.questions)[0]][0]
  return `${safeId()}_${slugify(firstQuestion)
    .replace(/^_+/, '')
    .substring(0, 50)
    .replace(/_+$/, '')}`
}

const normalizeQuestions = questions =>
  questions
    .map(q =>
      q
        .replace(/[\r\n]+/g, ' ')
        .replace(/\s+/g, ' ')
        .trim()
    )
    .filter(Boolean)

export default class Storage {
  private bp: typeof sdk
  private config
  private botId: string
  private categories: string[]

  constructor(bp: typeof sdk, config, botId) {
    this.bp = bp
    this.config = config
    this.botId = botId

    if (config.qnaCategories && config.qnaCategories.length > 0) {
      this.categories = config.qnaCategories
        .split(',')
        .map(x => x.trim())
        .filter(x => x.length)
    }
  }

  private async getAxiosConfig() {
    return this.bp.http.getAxiosConfigForBot(this.botId, { localUrl: true })
  }

  async initialize() {
    await this.migrate11To12()
    await this.syncQnaToNlu()
  }

  // @Deprecated > 12
  async migrate11To12() {
    const bot = await this.bp.bots.getBotById(this.botId)
    const qnas = await this.fetchQNAs()

    return await Promise.all(
      qnas.map(({ id, data }) => {
        const initial = _.cloneDeep(data)
        const questions = data.questions
        const answers = data.answers
        if (!data.category) {
          data.category = 'global'
        }
        if (_.isArray(questions)) {
          data.questions = {
            [bot.defaultLanguage]: questions
          }
        }
        if (_.isArray(answers)) {
          data.answers = {
            [bot.defaultLanguage]: answers
          }
        }

        if (!_.isEqual(initial, data)) {
          return this.update(data, id)
        }
      })
    )
  }

  // TODO Find better way to implement. When manually copying QNA, intents are not created.
  // Manual edit & save of each one is required for the intent to be created.
  async syncQnaToNlu() {
    const axiosConfig = await this.getAxiosConfig()
    const allQuestions = await this.fetchQNAs()
    const { data: allIntents } = await axios.get(`/mod/nlu/intents`, axiosConfig)

    const qnaItemsToSync = allQuestions.filter(
      qnaItem => qnaItem.data.enabled && !_.find(allIntents, i => i.name === getIntentId(qnaItem.id).toLowerCase())
    )

    return await Promise.mapSeries(qnaItemsToSync, async item => {
      // await this.checkForDuplicatedQuestions(item.data) // Why are we doing this check here?
      await this.createNLUIntentFromQnaItem(item)
    })
  }

  private async createNLUIntentFromQnaItem(qnaItem: QnaItem): Promise<void> {
    const axiosConfig = await this.getAxiosConfig()
    const utterances = {}
    for (const lang in qnaItem.data.questions) {
{'fraction_non_alphanumeric': 0.13297232250300842, 'fraction_numerical': 0.009025270758122744, 'mean_word_length': 2.71660145332588, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2251182', 'n_tokens_mistral': 2262, 'n_tokens_neox': 2174, 'n_words': 504}