<commit_before>import Promise from 'bluebird'
import _ from 'lodash'

import helpers from './helpers'

/*
  Possible options:
    - betweenGetAndSetCallback: will be called between the get and the set
    and wait for promise to resolve
    - tableName: overrides the KVS table's name
*/
module.exports = (knex, options = {}) => {
  const getSetCallback = options.betweenGetAndSetCallback || (() => Promise.resolve())
  const tableName = options.tableName || 'kvs'

  const get = (key, path) => {
    return knex(tableName).where({ key }).limit(1).then().get(0).then(row => {
      if (!row) {
        return null
      }

      const obj = JSON.parse(row.value)
      if (!path) {
        return obj
      }

      return _.at(obj, [path])[0]
    })
  }

  const _set = (key, value, operation = 'update') => {
    const now = helpers(knex).date.now()

    if (operation === 'update') {
      return knex(tableName).where({ key }).update({
        value: JSON.stringify(value),
        modified_on: now
      }).then()
    } else {
      return knex(tableName).insert({
        key: key,
        value: JSON.stringify(value),
        modified_on: now
      }).then()
    }
  }

  const set = (key, value, path) => {
    const setValue = obj => {
      if (path) {
        _.set(obj, path, value)
        return obj
      } else {
        return value
      }
    }

    return get(key)
      .then(original => {
        return getSetCallback()
          .then(() => {
            if (!_.isNil(original)) {
              const newObj = setValue(Object.assign({}, original))
              return _set(key, newObj, 'update')
            } else {
              const obj = setValue({})
              return _set(key, obj, 'insert')
                .catch(() => _set(key, obj, 'update'))
            }
          })
      })
  }

  const bootstrap = () => {
    return helpers(knex).createTableIfNotExists(tableName, function(table) {
      table.string('key').primary()
      table.text('value')
      table.timestamp('modified_on')
    })
  }

  return { get, set, bootstrap }
}<commit_msg>Use upsert to set KVs<commit_after>import Promise from 'bluebird'
import _ from 'lodash'

import helpers from './helpers'

/*
  Possible options:
    - betweenGetAndSetCallback: will be called between the get and the set
    and wait for promise to resolve
    - tableName: overrides the KVS table's name
*/
module.exports = (knex, options = {}) => {
  const getSetCallback = options.betweenGetAndSetCallback || (() => Promise.resolve())
  const tableName = options.tableName || 'kvs'

  const upsert = (key, value) => {
    let sql

    const params = {
      tableName,
      keyCol: 'key',
      valueCol: 'value',
      modifiedOnCol: 'modified_on',
      key,
      value: JSON.stringify(value),
      now: helpers(knex).date.now()
    }

    if (helpers(knex).isLite()) {
      sql = `
        INSERT OR REPLACE INTO :tableName: (:keyCol:, :valueCol:, :modifiedOnCol:)
        VALUES (:key, :value, :now)
      `
    } else {
      sql = `
        INSERT INTO :tableName: (:keyCol:, :valueCol:, :modifiedOnCol:)
        VALUES (:key, :value, :now)
        ON CONFLICT (:keyCol:) DO UPDATE
          SET :valueCol: = :value, :modifiedOnCol: = :now
      `
    }

    return knex.raw(sql, params)
  }

  const get = (key, path) => {
    return knex(tableName).where({ key }).limit(1).then().get(0).then(row => {
      if (!row) {
        return null
      }

      const obj = JSON.parse(row.value)
      if (!path) {
        return obj
      }

      return _.at(obj, [path])[0]
    })
  }

  const set = (key, value, path) => {
    if (!path) {
      return upsert(key, value)
    }

    const setValue = obj => {
      if (path) {
        _.set(obj, path, value)
        return obj
      } else {
        return value
      }
    }

    return get(key)
      .then(original => {
        return getSetCallback()
          .then(() => {
            if (!_.isNil(original)) {
              return upsert(key, setValue(Object.assign({}, original)))
            } else {
              return upsert(key, setValue({}))
            }
          })
      })
  }

  const bootstrap = () => {
    return helpers(knex).createTableIfNotExists(tableName, function(table) {
      table.string('key').primary()
      table.text('value')
      table.timestamp('modified_on')
    })
  }

  return { get, set, bootstrap }
}
{'fraction_non_alphanumeric': 0.14623607004775985, 'fraction_numerical': 0.001364566750056857, 'mean_word_length': 2.07122905027933, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '876361', 'n_tokens_mistral': 1405, 'n_tokens_neox': 1362, 'n_words': 340}