<commit_before>var async = require('async');
var through = require('through2');
var gutil = require('gulp-util');
var crypto = require('crypto');
var path = require('path');
var glob = require('glob');
var fs = require('fs');
var fsPath = require('fs-path');

Date.prototype.withoutTime = function () {
    var d = new Date(this);
    d.setHours(0,0,0,0);
    return d
}

var PluginError = gutil.PluginError;

// consts
const PLUGIN_NAME = 'gulp-attire';
const ALGORITHM = 'aes128';
const HASH = 'd6F3Efeq';
const TIMESTAMP = new Date().getTime().toString();

function encrypt(text){
  var cipher = crypto.createCipher(ALGORITHM, HASH)
  var crypted = cipher.update(text,'utf8','hex')
  crypted += cipher.final('hex');
  return crypted;
}

function decrypt(text){
  var decipher = crypto.createDecipher(ALGORITHM, HASH)
  var dec = decipher.update(text,'hex','utf8')
  dec += decipher.final('utf8');
  return dec;
}

function mergeFilesContent(files, cwd=null){
  output = '';
  async.each(files, function(file){
    var content = fs.readFileSync(path.resolve(cwd, file), 'utf8');
    output += content + '\n';
  }, function(err){
    if (err) {
      throw err;
    }
  });
  return output;
}

function createOutput(assets, cwd) {
  return new Promise(function(resolve, reject) {
    if (typeof assets == 'string') {
      glob(path.resolve(cwd, assets), {}, function (err, files) {
        if (err) {
          reject(err);
        } else {
          resolve(mergeFilesContent(files));
        }
      });
    }
    else if (assets instanceof Array) {
      resolve(mergeFilesContent(assets, cwd));
    }
    else {
      reject('createOutput: No type allowed');
    }
  });
}

function streamerParser(file, assetDir, config) {
  var stream = through({objectMode:true}, function(chunk, enc, callback){
    var self   = this;
    var string = chunk.toString()
    var tree   = JSON.parse(string);
    var hash   = encrypt(TIMESTAMP).substring(0,8);
    var parsed = {};

    async.forEachOf(tree, function(files, assetFileName, cb){
      createOutput(files, assetDir).then(function(data){
        var assetFile = path.parse(assetFileName);
        var outputName = assetFile.name + '-' + hash + assetFile.ext;
        var outputDir = (config.output)? config.output : 'public';
        parsed[assetFileName] = path.normalize(outputDir + '/' + assetFile.ext.replace('.', '') + '/' + outputName);
        if (! config.debug) {
          fsPath.writeFile(parsed[assetFileName], data, 'utf8', function(err) {
            if (err) {
              gutil.log(err)
              self.emit('error', new PluginError(PLUGIN_NAME, err));
            }
            cb();
          });
        } else {
          cb();
        }
      });
    }, function(err){
      if (err) {
        self.emit('error', new PluginError(PLUGIN_NAME, err));
      }
      self.push(JSON.stringify(parsed));
      callback();
    });
  });
  // stream.write('');
  return stream;
}

// plugin level function (dealing with files)
function gulpAttire(config={}) {
  // creating a stream through which each file will pass
  var stream = through.obj(function(file, enc, callback) {
    if (file.isBuffer()) {
      this.emit('error', new PluginError(PLUGIN_NAME, 'Buffers not supported!'));<commit_msg>Format the json output<commit_after>var async = require('async');
var through = require('through2');
var gutil = require('gulp-util');
var crypto = require('crypto');
var path = require('path');
var glob = require('glob');
var fs = require('fs');
var fsPath = require('fs-path');

Date.prototype.withoutTime = function () {
    var d = new Date(this);
    d.setHours(0,0,0,0);
    return d
}

var PluginError = gutil.PluginError;

// consts
const PLUGIN_NAME = 'gulp-attire';
const ALGORITHM = 'aes128';
const HASH = 'd6F3Efeq';
const TIMESTAMP = new Date().getTime().toString();

function encrypt(text){
  var cipher = crypto.createCipher(ALGORITHM, HASH)
  var crypted = cipher.update(text,'utf8','hex')
  crypted += cipher.final('hex');
  return crypted;
}

function decrypt(text){
  var decipher = crypto.createDecipher(ALGORITHM, HASH)
  var dec = decipher.update(text,'hex','utf8')
  dec += decipher.final('utf8');
  return dec;
}

function mergeFilesContent(files, cwd=null){
  output = '';
  async.each(files, function(file){
    var content = fs.readFileSync(path.resolve(cwd, file), 'utf8');
    output += content + '\n';
  }, function(err){
    if (err) {
      throw err;
    }
  });
  return output;
}

function createOutput(assets, cwd) {
  return new Promise(function(resolve, reject) {
    if (typeof assets == 'string') {
      glob(path.resolve(cwd, assets), {}, function (err, files) {
        if (err) {
          reject(err);
        } else {
          resolve(mergeFilesContent(files));
        }
      });
    }
    else if (assets instanceof Array) {
      resolve(mergeFilesContent(assets, cwd));
    }
    else {
      reject('createOutput: No type allowed');
    }
  });
}

function streamerParser(file, assetDir, config) {
  var stream = through({objectMode:true}, function(chunk, enc, callback){
    var self   = this;
    var string = chunk.toString()
    var tree   = JSON.parse(string);
    var hash   = encrypt(TIMESTAMP).substring(0,8);
    var parsed = {};

    async.forEachOf(tree, function(files, assetFileName, cb){
      createOutput(files, assetDir).then(function(data){
        var assetFile = path.parse(assetFileName);
        var outputName = assetFile.name + '-' + hash + assetFile.ext;
        var outputDir = (config.output)? config.output : 'public';
        parsed[assetFileName] = path.normalize(outputDir + '/' + assetFile.ext.replace('.', '') + '/' + outputName);
        if (! config.debug) {
          fsPath.writeFile(parsed[assetFileName], data, 'utf8', function(err) {
            if (err) {
              gutil.log(err)
              self.emit('error', new PluginError(PLUGIN_NAME, err));
            }
            cb();
          });
        } else {
          cb();
        }
      });
    }, function(err){
      if (err) {
        self.emit('error', new PluginError(PLUGIN_NAME, err));
      }
      self.push(JSON.stringify(parsed, null, 2));
      callback();
    });
  });
  // stream.write('');
  return stream;
}

// plugin level function (dealing with files)
function gulpAttire(config={}) {
  // creating a stream through which each file will pass
  var stream = through.obj(function(file, enc, callback) {
    if (file.isBuffer()) {
      this.emit('error', new PluginError(PLUGIN_NAME, 'Buffers not supported!'));
{'fraction_non_alphanumeric': 0.1568986568986569, 'fraction_numerical': 0.005341880341880342, 'mean_word_length': 3.0202453987730062, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6280359', 'n_tokens_mistral': 2089, 'n_tokens_neox': 2067, 'n_words': 474}