<commit_before>'use strict'

var fs = require('fs')
var miss = require('mississippi')
var config = require('../config')

var saveJobError = miss.through(function (chunck, encoding, callback) {
  var item = JSON.parse(chunck)
  var fileName = config.ERROR_DIRECTORY_PATH + '/' + item._id + '.json'

  if (item.errors.length > 0) {
    console.log(item._id + ': save-job-error')
    fs.writeFileSync(fileName, JSON.stringify(item, null, 2))
  }

  return callback(null, JSON.stringify(item))
})

module.exports = saveJobError<commit_msg>Moves pdfs to error folder<commit_after>'use strict'

var fs = require('fs')
var miss = require('mississippi')
var config = require('../config')
var path = require('path')

var saveJobError = miss.through(function (chunck, encoding, callback) {
  var item = JSON.parse(chunck)
  var fileName = config.ERROR_DIRECTORY_PATH + '/' + item._id + '.json'

  if (item.errors.length > 0) {
    console.log(item._id + ': save-job-error')
    console.log(item._id + ': save-job-error: Writes ' + item._id + '.json')
    fs.writeFileSync(fileName, JSON.stringify(item, null, 2))

    var documents = item.documents || []
    documents.push(item.restrictedAddressNotification)
    documents.push(item.noGuardianFoundNotification)
    documents.forEach(function (document) {
      console.log(item._id + ': save-job-error. Moves ' + document.document)
      var errorFile = config.ERROR_DIRECTORY_PATH + '/' + path.basename(document.document)
      fs.renameSync(document.document, errorFile)
    })
  }

  return callback(null, JSON.stringify(item))
})

module.exports = saveJobError
{'fraction_non_alphanumeric': 0.14819427148194272, 'fraction_numerical': 0.0024906600249066002, 'mean_word_length': 4.1506410256410255, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1266856', 'n_tokens_mistral': 525, 'n_tokens_neox': 511, 'n_words': 106}