<commit_before>var AJV = require('ajv')
var crypto = require('crypto')
var ecb = require('ecb')
var encoding = require('./encoding')
var flushWriteStream = require('flush-write-stream')
var fs = require('fs')
var mkdirp = require('mkdirp')
var path = require('path')
var pump = require('pump')
var readKeypair = require('./keypair/read')
var runParallel = require('run-parallel')
var runSeries = require('run-series')
var schema = require('./schemas/publication')
var sodium = require('sodium-prebuilt').api
var stringify = require('json-stable-stringify')
var through2 = require('through2')
var uuid = require('uuid/v4')

var validate = new AJV({allErrors: true}).compile(schema)

// TODO: Delete attachment files in /tmp for invalid publications.

// Create a writable stream that accepts one or more attachment chunks,
// followed by a publication chunk, and writes files and a signed
// timestamp to the data directory.
//
// Attachment chunks look like:
//
//     {
//       type: 'attachment',
//       stream: Readable,
//       filename: String,
//       encoding: String,
//       mimetype: String
//     }
//
// Publication fields contain JSON data.
module.exports = function (configuration, log, callback) {
  var directory = configuration.directory
  var attachments = []
  var validPublication = false
  return flushWriteStream.obj(
    function (object, _, done) {
      if (object.type === 'attachment') {
        writeAttachment(object, done)
      } else {
        writePublication(object, ecb(done, function (digest) {
          console.log('setting digest')
          validPublication = digest
          done()
        }))
      }
    },
    function (done) {
      if (validPublication) {
        callback(validPublication)
      } else {
        runParallel(
          attachments.map(function (attachment) {
            return function unlinkTemporaryFile (done) {
              fs.unlink(attachment.temporaryFile, function (error) {
                log.error(error)
                done()
              })
            }
          }),
          function () {
            done()
          }
        )<commit_msg>Remove console.log call<commit_after>var AJV = require('ajv')
var crypto = require('crypto')
var ecb = require('ecb')
var encoding = require('./encoding')
var flushWriteStream = require('flush-write-stream')
var fs = require('fs')
var mkdirp = require('mkdirp')
var path = require('path')
var pump = require('pump')
var readKeypair = require('./keypair/read')
var runParallel = require('run-parallel')
var runSeries = require('run-series')
var schema = require('./schemas/publication')
var sodium = require('sodium-prebuilt').api
var stringify = require('json-stable-stringify')
var through2 = require('through2')
var uuid = require('uuid/v4')

var validate = new AJV({allErrors: true}).compile(schema)

// TODO: Delete attachment files in /tmp for invalid publications.

// Create a writable stream that accepts one or more attachment chunks,
// followed by a publication chunk, and writes files and a signed
// timestamp to the data directory.
//
// Attachment chunks look like:
//
//     {
//       type: 'attachment',
//       stream: Readable,
//       filename: String,
//       encoding: String,
//       mimetype: String
//     }
//
// Publication fields contain JSON data.
module.exports = function (configuration, log, callback) {
  var directory = configuration.directory
  var attachments = []
  var validPublication = false
  return flushWriteStream.obj(
    function (object, _, done) {
      if (object.type === 'attachment') {
        writeAttachment(object, done)
      } else {
        writePublication(object, ecb(done, function (digest) {
          validPublication = digest
          done()
        }))
      }
    },
    function (done) {
      if (validPublication) {
        callback(validPublication)
      } else {
        runParallel(
          attachments.map(function (attachment) {
            return function unlinkTemporaryFile (done) {
              fs.unlink(attachment.temporaryFile, function (error) {
                log.error(error)
                done()
              })
            }
          }),
          function () {
            done()
          }
        )
{'fraction_non_alphanumeric': 0.12464589235127478, 'fraction_numerical': 0.00141643059490085, 'mean_word_length': 2.569502948609941, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6360170', 'n_tokens_mistral': 1156, 'n_tokens_neox': 1134, 'n_words': 328}