<commit_before>const q = require('q')
const EventEmitter = require('events').EventEmitter
const { MongoClient, ObjectId } = require('mongodb')
const Redis = require('redis')
const fs = require('fs')

const DATABASE_VERSION = 2

module.exports = function (config) {
  Object.assign(exports, config.common.storage)
  return function storageConnect () {
    if (exports._connected) {
      return q.when()
    }
    let uri

    if (config.mongo.uri) {
      uri = config.mongo.uri
    } else {
      uri = `mongodb://${config.mongo.host}:${config.mongo.port}/${config.mongo.database}`
    }
    delete config.mongo.host
    delete config.mongo.port
    delete config.mongo.database

    let redis = Redis.createClient(config.redis)
    let pub = Redis.createClient(config.redis)
    let sub = Redis.createClient(config.redis)

    let mongo = q.ninvoke(MongoClient, 'connect', uri, Object.assign({ promiseLibrary: Promise }, config.mongo))
      .then(db => {
        function wrapCollection (collection, cname) {
          let wrap = {}
          function keyToId (obj) {
            let idRegex = /^[a-f0-9]{24}$/
            if (obj instanceof Array) return obj.map(keyToId)
            if (obj._id && obj._id.$in) {
              return Object.assign({}, obj, {
                _id: {
                  $in: obj._id.$in.map(i => {
                    if (typeof i === 'string' && i.match(idRegex)) {
                      i = new ObjectId(i)
                    }
                    return i
                  })
                }
              })
            }
            if (typeof obj._id === 'string' && obj._id.match(idRegex)) {
              return Object.assign({}, obj, { _id: new ObjectId(obj._id) })
            }
            return obj
          }
          function idToKey (obj) {
            if (obj instanceof Array) return obj.map(idToKey)
            if (obj && obj._id) {
              obj._id = obj._id.toString()
            }
            return obj
          }
          ;['find', 'findOne', 'findEx', 'by', 'count', 'ensureIndex', 'remove', 'insert', 'update'].forEach(method => {
            wrap[method] = (...a) => {
              try {
                let orig = a[0]
                if (typeof a[0] === 'object') {
                  a[0] = keyToId(a[0])
                }
                if (method === 'update') {
                  a[2] = a[2] || {}
                  if (a[2].multi !== false && Object.keys(a[1]).reduce((l, v) => l && v[0] === '$', true)) {
                    a[2].multi = true
                  }
                  if (a[2].$merge) {
                    let merge = a[2].$merge
                    delete a[2].$merge
                    let flat = (obj, stack = []) => {
                      const ret = {}
                      if (typeof obj === 'object' && !Array.isArray(obj)) {
                        Object.entries(obj).forEach(([k, v]) => {
                          Object.assign(ret, flat(v, [...stack, k]))
                        })
                      } else if (stack.length) {
                        ret[stack.join('.')] = obj
                      } else {
                        return obj
                      }
                      return ret
                    }
                    a[2].$set = flat({}, merge)
                  }
                }
                let ex = method === 'findEx'
                if (ex) method = 'find'
                if (method.slice(0, 6) === 'insert') method = Array.isArray(a[0]) ? 'insertMany' : 'insertOne'

                let chain = collection[method](...a)
                if (method === 'insertOne') {
                  chain = chain.then((n) => {<commit_msg>Use the correct index for $merge<commit_after>const q = require('q')
const EventEmitter = require('events').EventEmitter
const { MongoClient, ObjectId } = require('mongodb')
const Redis = require('redis')
const fs = require('fs')

const DATABASE_VERSION = 2

module.exports = function (config) {
  Object.assign(exports, config.common.storage)
  return function storageConnect () {
    if (exports._connected) {
      return q.when()
    }
    let uri

    if (config.mongo.uri) {
      uri = config.mongo.uri
    } else {
      uri = `mongodb://${config.mongo.host}:${config.mongo.port}/${config.mongo.database}`
    }
    delete config.mongo.host
    delete config.mongo.port
    delete config.mongo.database

    let redis = Redis.createClient(config.redis)
    let pub = Redis.createClient(config.redis)
    let sub = Redis.createClient(config.redis)

    let mongo = q.ninvoke(MongoClient, 'connect', uri, Object.assign({ promiseLibrary: Promise }, config.mongo))
      .then(db => {
        function wrapCollection (collection, cname) {
          let wrap = {}
          function keyToId (obj) {
            let idRegex = /^[a-f0-9]{24}$/
            if (obj instanceof Array) return obj.map(keyToId)
            if (obj._id && obj._id.$in) {
              return Object.assign({}, obj, {
                _id: {
                  $in: obj._id.$in.map(i => {
                    if (typeof i === 'string' && i.match(idRegex)) {
                      i = new ObjectId(i)
                    }
                    return i
                  })
                }
              })
            }
            if (typeof obj._id === 'string' && obj._id.match(idRegex)) {
              return Object.assign({}, obj, { _id: new ObjectId(obj._id) })
            }
            return obj
          }
          function idToKey (obj) {
            if (obj instanceof Array) return obj.map(idToKey)
            if (obj && obj._id) {
              obj._id = obj._id.toString()
            }
            return obj
          }
          ;['find', 'findOne', 'findEx', 'by', 'count', 'ensureIndex', 'remove', 'insert', 'update'].forEach(method => {
            wrap[method] = (...a) => {
              try {
                let orig = a[0]
                if (typeof a[0] === 'object') {
                  a[0] = keyToId(a[0])
                }
                if (method === 'update') {
                  a[2] = a[2] || {}
                  if (a[2].multi !== false && Object.keys(a[1]).reduce((l, v) => l && v[0] === '$', true)) {
                    a[2].multi = true
                  }
                  if (a[0].$merge) {
                    let merge = a[0].$merge
                    delete a[0].$merge
                    let flat = (obj, stack = []) => {
                      const ret = {}
                      if (typeof obj === 'object' && !Array.isArray(obj)) {
                        Object.entries(obj).forEach(([k, v]) => {
                          Object.assign(ret, flat(v, [...stack, k]))
                        })
                      } else if (stack.length) {
                        ret[stack.join('.')] = obj
                      } else {
                        return obj
                      }
                      return ret
                    }
                    a[0].$set = flat({}, merge)
                  }
                }
                let ex = method === 'findEx'
                if (ex) method = 'find'
                if (method.slice(0, 6) === 'insert') method = Array.isArray(a[0]) ? 'insertMany' : 'insertOne'

                let chain = collection[method](...a)
                if (method === 'insertOne') {
                  chain = chain.then((n) => {
{'fraction_non_alphanumeric': 0.1475010158472166, 'fraction_numerical': 0.0059596370039279424, 'mean_word_length': 1.2818294190358468, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '623205', 'n_tokens_mistral': 2141, 'n_tokens_neox': 2046, 'n_words': 461}