<commit_before>/*
 * lib/processor.js
 *
 */

var _ = require("lodash");
var fs = require("fs-extra");
var async = require("async");
var stream = require("string-to-stream");
var zlib = require("zlib");


//-- export
module.exports = Processor;

//==============================================================================

function Processor(options) {
  var _options = _.merge({}, options);

  Object.defineProperty(this, "_options", {
    get: function() {
      return _options;
    }
  });
}

//------------------------------------------------------------------------------

Processor.prototype.export = function(data, done) {
  var self = this;
  var destFile = _.get(self._options, "to") || "";

  var output = fs.createWriteStream(destFile);
  var input = stream(data);
  if (/\.gz$/i.test(destFile)) {
    input = input.pipe(zlib.createGzip());
  }
  input.pipe(output);<commit_msg>create destination directory if it doesn't exist<commit_after>/*
 * lib/processor.js
 *
 */

var _ = require("lodash");
var fs = require("fs-extra");
var path = require("path");
var async = require("async");
var stream = require("string-to-stream");
var zlib = require("zlib");


//-- export
module.exports = Processor;

//==============================================================================

function Processor(options) {
  var _options = _.merge({}, options);

  Object.defineProperty(this, "_options", {
    get: function() {
      return _options;
    }
  });
}

//------------------------------------------------------------------------------

Processor.prototype.export = function(data, done) {
  var self = this;
  var destFile = _.get(self._options, "to") || "";

  try {
    fs.ensureDirSync(path.dirname(destFile));
  } catch (err) {
    return done(err);
  }

  var output = fs.createWriteStream(destFile);
  var input = stream(data);
  if (/\.gz$/i.test(destFile)) {
    input = input.pipe(zlib.createGzip());
  }
  input.pipe(output);
{'fraction_non_alphanumeric': 0.3130122950819672, 'fraction_numerical': 0.0, 'mean_word_length': 2.7557692307692307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7102050', 'n_tokens_mistral': 577, 'n_tokens_neox': 556, 'n_words': 118}