<commit_before>'use strict'

var elasticsearch = require('elasticsearch')
var _ = require('lodash')
var moment = require('moment')
var chalk = require('chalk')

function Importer (options) {
  this.options = options

  this.client = new elasticsearch.Client({
    host: options.host,
    log: options.log,
    requestTimeout: options.requestTimeout
  })
}

Importer.prototype.import = function (data, cb) {
  var self = this

  var body = []
  if (!_.isArray(data)) {
    data = [ data ]
  }

  data.map(function (record) {
    Object.keys(record).map(function (key) {
      if (_.isObject(record[ key ])) {
        if (record[ key ][ '$oid' ]) {
          record[ key ] = record[ key ][ '$oid' ]
        } else if (record[ key ][ '$date' ]) {
          record[ key ] = moment(record[ key ][ '$date' ]).toDate()
        }
      }
    })

    if (self.options.ignore) {
      self.options.ignore.split(',').map(function (ignore) {
        ignore = ignore.trim()
        if (ignore.indexOf('[*].') !== -1) {
          var field = ignore.substring(0, ignore.indexOf('[*]'))
          var obj = record[ field ]
          if (obj && _.isArray(obj)) {
            var afterField = ignore.substring(ignore.indexOf('[*].') + 4)
            obj.map(function (value) {
              _.unset(value, afterField)
            })
          }
        } else {
          _.unset(record, ignore)
        }
      })
    }

    if (self.options.transform) {
      record = self.options.transform(record) || record
    }

    body.push({ index: { _index: self.options.index, _type: self.options.type, _id: record._id } })
    delete record._id
    body.push(record)
  })

  self.client.bulk({ body: body }, function (err, resp) {
    if (err || resp.errors) {
      if (cb) {
        return cb(err || resp.errors)
      }

      var color = self.options.warnErrors ? chalk.yellow : chalk.red
      console.log(color('Elastic Import: Error importing data'))

      if (err) {
        console.log(err)
      } else {
        var errors = _.filter(resp.items, function (item) {
          return (item.index && item.index.error) || (item.create && item.create.error)
        })

        errors.map(function (item) {
          var message = item.index ? item.index.error : item.create.error
          console.log(
            color('Error: ' + message.type),
            color('-'), color('Reason: ' + message.reason),
            message.caused_by ? color('- Caused by: ' + message.caused_by.reason) : ''<commit_msg>Remove _id if not present<commit_after>'use strict'

var elasticsearch = require('elasticsearch')
var _ = require('lodash')
var moment = require('moment')
var chalk = require('chalk')

function Importer (options) {
  this.options = options

  this.client = new elasticsearch.Client({
    host: options.host,
    log: options.log,
    requestTimeout: options.requestTimeout
  })
}

Importer.prototype.import = function (data, cb) {
  var self = this

  var body = []
  if (!_.isArray(data)) {
    data = [ data ]
  }

  data.map(function (record) {
    Object.keys(record).map(function (key) {
      if (_.isObject(record[ key ])) {
        if (record[ key ][ '$oid' ]) {
          record[ key ] = record[ key ][ '$oid' ]
        } else if (record[ key ][ '$date' ]) {
          record[ key ] = moment(record[ key ][ '$date' ]).toDate()
        }
      }
    })

    if (self.options.ignore) {
      self.options.ignore.split(',').map(function (ignore) {
        ignore = ignore.trim()
        if (ignore.indexOf('[*].') !== -1) {
          var field = ignore.substring(0, ignore.indexOf('[*]'))
          var obj = record[ field ]
          if (obj && _.isArray(obj)) {
            var afterField = ignore.substring(ignore.indexOf('[*].') + 4)
            obj.map(function (value) {
              _.unset(value, afterField)
            })
          }
        } else {
          _.unset(record, ignore)
        }
      })
    }

    if (self.options.transform) {
      record = self.options.transform(record) || record
    }
    
    if (record._id) {
      body.push({ index: { _index: self.options.index, _type: self.options.type, _id: record._id } })
      delete record._id
    } else {
      body.push({ index: { _index: self.options.index, _type: self.options.type } })
    }
    body.push(record)
  })

  self.client.bulk({ body: body }, function (err, resp) {
    if (err || resp.errors) {
      if (cb) {
        return cb(err || resp.errors)
      }

      var color = self.options.warnErrors ? chalk.yellow : chalk.red
      console.log(color('Elastic Import: Error importing data'))

      if (err) {
        console.log(err)
      } else {
        var errors = _.filter(resp.items, function (item) {
          return (item.index && item.index.error) || (item.create && item.create.error)
        })

        errors.map(function (item) {
          var message = item.index ? item.index.error : item.create.error
          console.log(
            color('Error: ' + message.type),
            color('-'), color('Reason: ' + message.reason),
            message.caused_by ? color('- Caused by: ' + message.caused_by.reason) : ''
{'fraction_non_alphanumeric': 0.16029641185647425, 'fraction_numerical': 0.001170046801872075, 'mean_word_length': 2.164096236890808, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2525652', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1586, 'n_words': 353}