<commit_before>var uuid = require('cuid')
var pumpify = require('pumpify')
var through = require('through2')
var debug = require('debug')('lib/import')
var parseInputStream = require('../lib/util/parse-input-stream.js')

var COMPOUND_KEY_SEPARATOR = '+'

module.exports = function (db, opts) {
  if (!opts) opts = {}
  if (!opts.dataset) throw new Error('Error: Must specify dataset (-d)')
  if (opts.dataset === 'files') throw new Error('Cannot import into the \'files\' dataset')

  var transform = through.obj(function (obj, enc, next) {
    debug('heres my obj!', obj)

    var key
    if (typeof opts.key === 'object' && opts.key.length > 1) {
      key = opts.key.sort().map(function (key) {
        return obj[key] || ''
      }).join(COMPOUND_KEY_SEPARATOR)
    } else {
      key = obj[opts.key] || obj.key<commit_msg>use isArray instead of === 'object'<commit_after>var uuid = require('cuid')
var pumpify = require('pumpify')
var through = require('through2')
var debug = require('debug')('lib/import')
var parseInputStream = require('../lib/util/parse-input-stream.js')

var COMPOUND_KEY_SEPARATOR = '+'

module.exports = function (db, opts) {
  if (!opts) opts = {}
  if (!opts.dataset) throw new Error('Error: Must specify dataset (-d)')
  if (opts.dataset === 'files') throw new Error('Cannot import into the \'files\' dataset')

  var transform = through.obj(function (obj, enc, next) {
    debug('heres my obj!', obj)

    var key
    if (Array.isArray(opts.key) && opts.key.length > 1) {
      key = opts.key.sort().map(function (key) {
        return obj[key] || ''
      }).join(COMPOUND_KEY_SEPARATOR)
    } else {
      key = obj[opts.key] || obj.key
{'fraction_non_alphanumeric': 0.172950329144225, 'fraction_numerical': 0.0023937761819269898, 'mean_word_length': 3.696629213483146, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1592491', 'n_tokens_mistral': 551, 'n_tokens_neox': 539, 'n_words': 145}