<commit_before>exec = require('child_process').exec
minimatch = require 'minimatch'
fstream = require 'fstream'
chalk = require 'chalk'
tmp = require 'tmp'
fs = require 'fs'
path = require 'path'
async = require 'async'
archiver = require 'archiver'
readDirSyncRecursive = require 'fs-readdir-recursive'
_ = require 'underscore'

IGNORE_FILE = '.versalignore'

module.exports =
  createBundle: (dir, callback) ->

    console.log chalk.yellow('Creating bundle.tar.gz')
    bundleFilesInFolder dir, (err, bundlePath) ->
      if err then return callback err

      getBundleSize bundlePath, (err, bundleSize) ->
        console.log chalk.grey "bundle path: #{bundlePath}"
        console.log chalk.grey "bundle size: #{bundleSize}KB"

        bundleStream = fs.createReadStream bundlePath
        callback null, bundleStream

getBundleSize = (bundlePath, callback) ->
  fs.stat bundlePath, (err, stats) ->
    if err then return callback err
    size = parseInt(stats.size / 1024, 10)
    callback null, size

bundleFilesInFolder = (dir, callback) ->
  tmp.dir (err, tmpdir) ->
    if err then return callback err

    createIgnoreFilter dir, (err, filter) ->
      if err then return callback err

      bundlePath = path.join tmpdir, 'bundle.tar.gz'
      bundleOutput = fs.createWriteStream bundlePath

      bundle = archiver 'tar',
        gzip: true
        gzipOptions:
          level: 1

      bundleFiles = _.select readDirSyncRecursive(dir), filter
      bundle.on 'error', callback
      bundle.pipe bundleOutput
<commit_msg>Cleanup + add missing error handler<commit_after>exec = require('child_process').exec
minimatch = require 'minimatch'
fstream = require 'fstream'
chalk = require 'chalk'
tmp = require 'tmp'
fs = require 'fs'
path = require 'path'
async = require 'async'
archiver = require 'archiver'
readDirSyncRecursive = require 'fs-readdir-recursive'
_ = require 'underscore'

IGNORE_FILE = '.versalignore'

module.exports =
  createBundle: (dir, callback) ->

    console.log chalk.yellow('Creating bundle.tar.gz')

    bundleFilesInFolder dir, (err, bundlePath) ->
      if err then return callback err

      getBundleSize bundlePath, (err, bundleSize) ->
        if err then return callback err

        console.log chalk.grey "bundle path: #{bundlePath}"
        console.log chalk.grey "bundle size: #{bundleSize}KB"

        bundleStream = fs.createReadStream bundlePath
        callback null, bundleStream

getBundleSize = (bundlePath, callback) ->
  fs.stat bundlePath, (err, stats) ->
    if err then return callback err

    size = parseInt(stats.size / 1024, 10)
    callback null, size

bundleFilesInFolder = (dir, callback) ->
  tmp.dir (err, tmpdir) ->
    if err then return callback err

    createIgnoreFilter dir, (err, filter) ->
      if err then return callback err

      bundlePath = path.join tmpdir, 'bundle.tar.gz'
      bundleOutput = fs.createWriteStream bundlePath

      bundle = archiver 'tar',
        gzip: true
        gzipOptions:
          level: 1

      bundleFiles = _.select readDirSyncRecursive(dir), filter
      bundle.on 'error', callback
      bundle.pipe bundleOutput

{'fraction_non_alphanumeric': 0.09977685686962065, 'fraction_numerical': 0.004462862607586867, 'mean_word_length': 3.2578018995929443, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '462735', 'n_tokens_mistral': 956, 'n_tokens_neox': 904, 'n_words': 282}