<commit_before>var derp = require('./derp'),
  glob = require('glob'),
  fs = require('fs'),
  path = require('path'),
  JSZip = require('jszip'),
  Minimatch = require('minimatch').Minimatch,
  isWindows = require('os').platform() === 'win32',
  async = require('async');

module.exports = function bundle(config, fn){
  // run npm install
  // create manifest
  // create zip and add all files to it
  // write zip to config.out
  derp.sync(config,
    // derp.run('npm install', config.src),
    module.exports.manifest, module.exports.patch, module.exports.zip, fn);
};

module.exports.patch = function(config, fn){
  var debug = require('debug')('lone:bundle:patch'),
    DiffMatchPatch = require('diff-match-patch'),
    dmp = new DiffMatchPatch();

  if(!isWindows){
    debug('not windows.  dont need to patch');
    return fn(null, config);
  }
  [
    [
      path.resolve(config.src + '/node_modules/bson/ext/index.js'),
      path.resolve(__dirname + '/../patch/bson.patch')
    ],
    [
      path.resolve(config.src + '/node_modules/mongodb/node_modules/kerberos/index.js'),
      path.resolve(__dirname + '/../patch/kerberos.patch')
    ]
  ].map(function(item){
    debug('applying', item[0], item[1]);
    var src = item[0],
      patch = item[1],
      p = dmp.patch_fromText(fs.readFileSync(patch, 'utf-8'));
    fs.writeFileSync(src, dmp.patch_apply(p, fs.readFileSync(src, 'utf-8')));
  });

  debug('complete');
  return fn(null, config);
};

module.exports.manifest = function bundle_manifest(config, fn){
  var debug = require('debug')('lone:bundle:manifest'),
    manifest = {files: [], directories: []},
    ignore = [];

  // Already ran the manifest
  if(config.manifest) return fn(null, config);

  function addIgnore(pattern){
    debug('ignore', pattern);
    ignore.push(new Minimatch(pattern, {flipNegate: true}));
  }
  config.ignore.push.apply(config.ignore, ['node_modules/lone/**']);

  config.ignore.map(addIgnore);
  glob('{*,**/*}', {cwd: config.src, root: config.src, mark: true, strict: true}, function (err, files){
    if(err) return fn(err);

    manifest.files = files.filter(function(p){
      var gIgnore = ignore.some(function(i){
        return i.match(p);
      });

      if(gIgnore === true) return false;
      if(p.charAt(p.length - 1) === '/'){
        manifest.directories.push(p);
        return false;
      }
      return true;
    });
    config.manifest = manifest;
    debug('files collected', config.manifest.files.length);

    fn(null, config);
  });
};

module.exports.zip = function bundle_zip(config, fn){
  var zip = new JSZip(null, {type: 'nodebuffer'}),
    debug = require('debug')('lone:bundle:zip');

  debug('reading files');

  async.parallel(config.manifest.files.map(function(p){
    return function(cb){
      fs.readFile(config.src + '/' + p, function(err, data){
        if(err) return cb(err);

        zip.file(p, data);
        debug('added', derp.relativity(p));
        cb(null);
      });
    };
  }), function(err){
    if(err) return fn(err);
    debug('writing', derp.relativity(config.bundle));
    fs.writeFile(config.bundle, zip.generate({type:'nodebuffer'}), function(err){
      if(err) return fn(err);
      fn(null, config);
    });
  });
};
<commit_msg>[#10] i hate diff make patch.<commit_after>var derp = require('./derp'),
  glob = require('glob'),
  fs = require('fs'),
  path = require('path'),
  JSZip = require('jszip'),
  Minimatch = require('minimatch').Minimatch,
  isWindows = require('os').platform() === 'win32',
  async = require('async');

module.exports = function bundle(config, fn){
  // run npm install
  // create manifest
  // create zip and add all files to it
  // write zip to config.out
  derp.sync(config,
    // derp.run('npm install', config.src),
    module.exports.manifest, module.exports.patch, module.exports.zip, fn);
};

module.exports.patch = function(config, fn){
  var debug = require('debug')('lone:bundle:patch'),
    DiffMatchPatch = require('diff-match-patch'),
    dmp = new DiffMatchPatch();

  // if(!isWindows){
  //   debug('not windows.  dont need to patch');
  //   return fn(null, config);
  // }
  [
    [
      path.resolve(config.src + '/node_modules/bson/ext/index.js'),
      path.resolve(__dirname + '/../patch/bson.patch')
    ],
    [
      path.resolve(config.src + '/node_modules/mongodb/node_modules/kerberos/index.js'),
      path.resolve(__dirname + '/../patch/kerberos.patch')
    ]
  ].map(function(item){
    debug('applying', item[0], item[1]);
    var src = item[0],
      patch = item[1],
      p = dmp.patch_fromText(fs.readFileSync(patch, 'utf-8'));
    var fresh = dmp.patch_apply(p, fs.readFileSync(src, 'utf-8'));
    debug('fresh', fresh[0]);
    fs.writeFileSync(src, fresh[0]);
  });

  debug('complete');
  return fn(null, config);
};

module.exports.manifest = function bundle_manifest(config, fn){
  var debug = require('debug')('lone:bundle:manifest'),
    manifest = {files: [], directories: []},
    ignore = [];

  // Already ran the manifest
  if(config.manifest) return fn(null, config);

  function addIgnore(pattern){
    debug('ignore', pattern);
    ignore.push(new Minimatch(pattern, {flipNegate: true}));
  }
  config.ignore.push.apply(config.ignore, ['node_modules/lone/**']);

  config.ignore.map(addIgnore);
  glob('{*,**/*}', {cwd: config.src, root: config.src, mark: true, strict: true}, function (err, files){
    if(err) return fn(err);

    manifest.files = files.filter(function(p){
      var gIgnore = ignore.some(function(i){
        return i.match(p);
      });

      if(gIgnore === true) return false;
      if(p.charAt(p.length - 1) === '/'){
        manifest.directories.push(p);
        return false;
      }
      return true;
    });
    config.manifest = manifest;
    debug('files collected', config.manifest.files.length);

    fn(null, config);
  });
};

module.exports.zip = function bundle_zip(config, fn){
  var zip = new JSZip(null, {type: 'nodebuffer'}),
    debug = require('debug')('lone:bundle:zip');

  debug('reading files');

  async.parallel(config.manifest.files.map(function(p){
    return function(cb){
      fs.readFile(config.src + '/' + p, function(err, data){
        if(err) return cb(err);

        zip.file(p, data);
        debug('added', derp.relativity(p));
        cb(null);
      });
    };
  }), function(err){
    if(err) return fn(err);
    debug('writing', derp.relativity(config.bundle));
    fs.writeFile(config.bundle, zip.generate({type:'nodebuffer'}), function(err){
      if(err) return fn(err);
      fn(null, config);
    });
  });
};

{'fraction_non_alphanumeric': 0.18639951426836673, 'fraction_numerical': 0.003339404978749241, 'mean_word_length': 3.314996725605763, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1144769', 'n_tokens_mistral': 2219, 'n_tokens_neox': 2159, 'n_words': 428}