<commit_before>import _ from 'lodash'
import crypto from 'crypto'

import zscore from 'zscore'
import natural from 'natural'

import Provider from './base'
import Entities from './entities'

const NATIVE_HASH_KVS_KEY = 'nlu/native/updateMetadata'
const NATIVE_MODEL = 'nlu/native/model'

export default class NativeProvider extends Provider {
  constructor(config) {
    super({ ...config, name: 'native', entityKey: '@native' })
    this.classifier = null
  }

  async init() {}

  async checkSyncNeeded() {
    const intents = await this.storage.getIntents()
    return !await this._isInSync(intents)
  }

  _getProjectName() {
    const scope = 'all'
    return `${this.env}__${this.project}__${scope}`
  }

  async _isInSync(localIntents) {
    const intentsHash = crypto
      .createHash('md5')
      .update(JSON.stringify(localIntents))
      .digest('hex')

    const metadata = await this.kvs.get(NATIVE_HASH_KVS_KEY)
    return metadata && metadata.hash === intentsHash
  }

  async _onSyncSuccess(localIntents) {
    const intentsHash = crypto
      .createHash('md5')
      .update(JSON.stringify(localIntents))
      .digest('hex')

    // We save the model hash and model to the KVS
    await this.kvs.set(NATIVE_HASH_KVS_KEY, { hash: intentsHash })
    await this.kvs.set(NATIVE_MODEL, JSON.stringify(this.classifier))
  }

  async _restoreModel() {
    const model = await this.kvs.get(NATIVE_MODEL)

    if (!model) {
      this.classifier = new natural.BayesClassifier()
    }

    this.classifier = natural.BayesClassifier.restore(JSON.parse(model))
  }

  async getCustomEntities() {
    // Native NLU doesn't support entity extraction
    return []
  }

  async sync() {
    const intents = await this.storage.getIntents()

    if (await this._isInSync(intents)) {
      this.logger.debug('[NLU::Native] Model is up to date')
      return
    } else {
      this.logger.debug('[NLU::Native] The model needs to be updated')
    }

    const classifier = new natural.BayesClassifier()

    let samples_count = 0

    intents.forEach(intent => {
      intent.utterances.forEach(utterance => {
        const extracted = this.parser.extractLabelsFromCanonical(utterance, intent.entities)
        samples_count += 1
        classifier.addDocument(extracted.text, intent.name)
      })
    })

    this.logger.debug(`[NLU::Native] Started training model from ${samples_count} samples`)

    try {
      classifier.train()
    } catch (err) {
      return this.logger.error(`[NLU::Native] Error training model: ${err.message}`)
    }

    this.classifier = classifier

    await this._onSyncSuccess(intents)

    this.logger.info(`[NLU::Native] Synced model`)
  }

  async extract(incomingEvent) {
    if (!this.classifier) {
      if (await this.checkSyncNeeded()) {
        await this.sync()
      } else {
        await this._restoreModel()
      }
    }

    const classifications = _.orderBy(this.classifier.getClassifications(incomingEvent.text), ['value'], ['desc'])

    let allScores = zscore(_.map(classifications, c => parseFloat(c.value)))
    allScores = allScores.map((s, i) => {
      return (
        s -
        (_.get(allScores, i + 1) || 0) * 0.5 -
        (_.get(allScores, i + 2) || 0) * 0.75 -
        (_.get(allScores, i + 3) || 0)
      )
    })

    const intents = _.map(classifications, (c, i) => {
      return {
        intent: c.label,
        score: allScores[i]
      }
    })

    const bestIntent = _.first(intents)

    const intentName = _.get(bestIntent, 'intent') || 'None'
    const confidence = _.get(bestIntent, 'score') || 0

    return {
      intent: {
        name: intentName,
        confidence: parseFloat(confidence),
        provider: 'native'<commit_msg>feat(nlu): added ability to provide custom stemmer<commit_after>import _ from 'lodash'
import crypto from 'crypto'

import zscore from 'zscore'
import natural from 'natural'

import Provider from './base'
import Entities from './entities'

const NATIVE_HASH_KVS_KEY = 'nlu/native/updateMetadata'
const NATIVE_MODEL = 'nlu/native/model'

export default class NativeProvider extends Provider {
  constructor(config) {
    super({ ...config, name: 'native', entityKey: '@native' })
    this.classifier = null
  }

  async init() {}

  async checkSyncNeeded() {
    const intents = await this.storage.getIntents()
    return !await this._isInSync(intents)
  }

  _getProjectName() {
    const scope = 'all'
    return `${this.env}__${this.project}__${scope}`
  }

  setStemmer(stemmer) {
    if (!stemmer) {
      this.customStemmer = null
    } else if (!_.isFunction(stemmer)) {
      this.logger.error('[NLU::Native] Stemmer must be a function')
      this.customStemmer = null
    } else {
      this.customStemmer = stemmer
    }
  }

  _stemText(text) {
    if (this.customStemmer) {
      return this.customStemmer(text)
    } else {
      return natural.PorterStemmer.tokenizeAndStem(text)
    }
  }

  async _isInSync(localIntents) {
    const intentsHash = crypto
      .createHash('md5')
      .update(JSON.stringify(localIntents))
      .digest('hex')

    const metadata = await this.kvs.get(NATIVE_HASH_KVS_KEY)
    return metadata && metadata.hash === intentsHash
  }

  async _onSyncSuccess(localIntents) {
    const intentsHash = crypto
      .createHash('md5')
      .update(JSON.stringify(localIntents))
      .digest('hex')

    // We save the model hash and model to the KVS
    await this.kvs.set(NATIVE_HASH_KVS_KEY, { hash: intentsHash })
    await this.kvs.set(NATIVE_MODEL, JSON.stringify(this.classifier))
  }

  async _restoreModel() {
    const model = await this.kvs.get(NATIVE_MODEL)

    if (!model) {
      this.classifier = new natural.BayesClassifier()
    }

    this.classifier = natural.BayesClassifier.restore(JSON.parse(model))
  }

  async getCustomEntities() {
    // Native NLU doesn't support entity extraction
    return []
  }

  async sync() {
    const intents = await this.storage.getIntents()

    if (await this._isInSync(intents)) {
      this.logger.debug('[NLU::Native] Model is up to date')
      return
    } else {
      this.logger.debug('[NLU::Native] The model needs to be updated')
    }

    const classifier = new natural.BayesClassifier()
    classifier.stemmer = { tokenizeAndStem: this._stemText }

    let samples_count = 0

    intents.forEach(intent => {
      intent.utterances.forEach(utterance => {
        const extracted = this.parser.extractLabelsFromCanonical(utterance, intent.entities)
        samples_count += 1
        classifier.addDocument(this._stemText(extracted.text), intent.name)
      })
    })

    this.logger.debug(`[NLU::Native] Started training model from ${samples_count} samples`)

    try {
      classifier.train()
    } catch (err) {
      return this.logger.error(`[NLU::Native] Error training model: ${err.message}`)
    }

    this.classifier = classifier

    await this._onSyncSuccess(intents)

    this.logger.info(`[NLU::Native] Synced model`)
  }

  async extract(incomingEvent) {
    if (!this.classifier) {
      if (await this.checkSyncNeeded()) {
        await this.sync()
      } else {
        await this._restoreModel()
      }
    }

    const classifications = _.orderBy(this.classifier.getClassifications(incomingEvent.text), ['value'], ['desc'])

    let allScores = zscore(_.map(classifications, c => parseFloat(c.value)))
    allScores = allScores.map((s, i) => {
      return (
        s -
        (_.get(allScores, i + 1) || 0) * 0.5 -
        (_.get(allScores, i + 2) || 0) * 0.75 -
        (_.get(allScores, i + 3) || 0)
      )
    })

    const intents = _.map(classifications, (c, i) => {
      return {
        intent: c.label,
        score: allScores[i]
      }
    })

    const bestIntent = _.first(_.orderBy(intents, ['score'], ['desc']))

    const intentName = _.get(bestIntent, 'intent') || 'None'
    const confidence = _.get(bestIntent, 'score') || 0

    return {
      intent: {
        name: intentName,
        confidence: parseFloat(confidence),
        provider: 'native'
{'fraction_non_alphanumeric': 0.12953045211109726, 'fraction_numerical': 0.003985552372649147, 'mean_word_length': 2.9037433155080214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '253715', 'n_tokens_mistral': 2670, 'n_tokens_neox': 2568, 'n_words': 559}