<commit_before>'use strict'

module.exports = function (argv) {
  try {
    var config = require(`${process.cwd()}/config.js`)
  } catch (err) {
    console.log('config.js missing! Please run "mailcap bootstrap"')
    process.exit()
  }
  var ReGrid = require('rethinkdb-regrid')
  var bucket = ReGrid(config.rethinkdb, config.regrid)
  var SMTPServer = require('smtp-server').SMTPServer
  var MailParser = require('mailparser').MailParser
  var co = require('co')
  var streamPromise = require('./mcUtil').streamPromise
  var _ = require('lodash')
  var r = require('rethinkdbdash')(config.rethinkdb)

  var onAuth = function (auth, session, cb) {
    return cb()
  }

  var onMailFrom = function (address, session, cb) {
    return cb()
  }

  var onRcptTo = function (address, session, cb) {
    if (address.address === config.address) {
      return cb()
    }

    return cb(new Error('DENIED'))
  }

  var onData = function (stream, session, cb) {
    let mailparser = new MailParser()

    stream.pipe(mailparser)

    mailparser.on('end', function (mail) {
      co(function *() {
        mail.attachments = mail.attachments || []
        let attachments = mail.attachments.map(function (attachment) {
          return _.omit(attachment, 'content')
        })
        let obj = {
          html: mail.html,
          text: mail.text,
          headers: mail.headers,
          subject: mail.subject,
          messageId: mail.messageId,
          createdAt: mail.date,

          from: _.compact(_.map(mail.from, (i) => _.isString(i.address) ? i.address.toLowerCase() : null)),
          to: _.compact(_.map(mail.to, (i) => _.isString(i.address) ? i.address.toLowerCase() : null)),
          cc: _.compact(_.map(mail.cc, (i) => _.isString(i.address) ? i.address.toLowerCase() : null)),

          attachments: attachments,
          addresses: _.uniq(obj.from.concat(obj.to, obj.cc))
        }

        let results = yield r.table(config.table).insert(obj, {returnChanges: true}).run()
        let email = results.changes[0].new_val
        let queue = []

        for (let attachment of mail.attachments) {
          let wstream = bucket.upload(`${email.id}/${attachment.fileName}`)
          queue.push(streamPromise(wstream))
          wstream.write(attachment.content)
          wstream.end()
        }

        yield Promise.all(queue)

        console.log('Received Mail:', email.addresses)
      }).catch(function (err) {
        console.log(err.stack)
      }).finally(function () {
        cb()
      })
    })
  }

  co(function *() {
    yield bucket.initBucket()

    var server = new SMTPServer({
      secure: false,
      hideSTARTTLS: true,
      name: 'MAIL CAPTURE SERVER',
      disabledCommands: ['AUTH'],
      onAuth: onAuth,
      onMailFrom: onMailFrom,
      onRcptTo: onRcptTo,
      onData: onData,
      logger: false
    })

    server.listen(config.port)

    console.log('Server Started...')
  })
}
<commit_msg>fix obj is not defined<commit_after>'use strict'

module.exports = function (argv) {
  try {
    var config = require(`${process.cwd()}/config.js`)
  } catch (err) {
    console.log('config.js missing! Please run "mailcap bootstrap"')
    process.exit()
  }
  var ReGrid = require('rethinkdb-regrid')
  var bucket = ReGrid(config.rethinkdb, config.regrid)
  var SMTPServer = require('smtp-server').SMTPServer
  var MailParser = require('mailparser').MailParser
  var co = require('co')
  var streamPromise = require('./mcUtil').streamPromise
  var _ = require('lodash')
  var r = require('rethinkdbdash')(config.rethinkdb)

  var onAuth = function (auth, session, cb) {
    return cb()
  }

  var onMailFrom = function (address, session, cb) {
    return cb()
  }

  var onRcptTo = function (address, session, cb) {
    if (address.address === config.address) {
      return cb()
    }

    return cb(new Error('DENIED'))
  }

  var onData = function (stream, session, cb) {
    let mailparser = new MailParser()

    stream.pipe(mailparser)

    mailparser.on('end', function (mail) {
      co(function *() {
        mail.attachments = mail.attachments || []
        let attachments = mail.attachments.map(function (attachment) {
          return _.omit(attachment, 'content')
        })
        let obj = {
          html: mail.html,
          text: mail.text,
          headers: mail.headers,
          subject: mail.subject,
          messageId: mail.messageId,
          createdAt: mail.date,

          from: _.compact(_.map(mail.from, (i) => _.isString(i.address) ? i.address.toLowerCase() : null)),
          to: _.compact(_.map(mail.to, (i) => _.isString(i.address) ? i.address.toLowerCase() : null)),
          cc: _.compact(_.map(mail.cc, (i) => _.isString(i.address) ? i.address.toLowerCase() : null)),

          attachments: attachments
        }

        obj.addresses = _.uniq(obj.from.concat(obj.to, obj.cc))

        let results = yield r.table(config.table).insert(obj, {returnChanges: true}).run()
        let email = results.changes[0].new_val
        let queue = []

        for (let attachment of mail.attachments) {
          let wstream = bucket.upload(`${email.id}/${attachment.fileName}`)
          queue.push(streamPromise(wstream))
          wstream.write(attachment.content)
          wstream.end()
        }

        yield Promise.all(queue)

        console.log('Received Mail:', email.addresses)
      }).catch(function (err) {
        console.log(err.stack)
      }).finally(function () {
        cb()
      })
    })
  }

  co(function *() {
    yield bucket.initBucket()

    var server = new SMTPServer({
      secure: false,
      hideSTARTTLS: true,
      name: 'MAIL CAPTURE SERVER',
      disabledCommands: ['AUTH'],
      onAuth: onAuth,
      onMailFrom: onMailFrom,
      onRcptTo: onRcptTo,
      onData: onData,
      logger: false
    })

    server.listen(config.port)

    console.log('Server Started...')
  })
}

{'fraction_non_alphanumeric': 0.14512855209742895, 'fraction_numerical': 0.00033829499323410016, 'mean_word_length': 2.5196428571428573, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6385206', 'n_tokens_mistral': 1878, 'n_tokens_neox': 1845, 'n_words': 394}