<commit_before>var async = require('async'),
  debug = require('debug')('mongoscope-ci'),
  request = require('request'),
  os = require('os'),
  path = require('path'),
  untildify = require('untildify'),
  fs = require('fs-extra'),
  zlib = require('zlib'),
  tar = require('tar'),
  spawn = require('child_process').spawn;

function platform(mongodb){
  var p = os.platform();
  if(mongodb && p === 'darwin') return 'osx';
  return p;
}

function arch(mongodb){
  var a = os.arch();
  if(mongodb && a === 'x64') return 'x86_64';
  return a;
}

function nodejs(){
  return function(fn){
    var debug = require('debug')('mongoscope-ci:nodejs'),
      basename = 'node-v'+config.nodejs+'-'+platform()+'-' + arch(),
      pkg = {
        name: 'nodejs',
        version: config.nodejs,
        artifact: basename+'.tar.gz',
        checksum: '',
        url: 'http://nodejs.org/dist/v'+config.nodejs+'/'+basename+'.tar.gz'
      };

    async.series([download(pkg), extract(pkg)], function(err, res){
      if(err) return fn(err);

      var bin = path.resolve(res[1] + '/bin');
      process.env.PATH = bin + ':' + process.env.PATH;
      debug('add PATH ' + bin);
      fn();
    });
  };
}

function mongodb(){
  return function(fn){
    var debug = require('debug')('mongoscope-ci:mongodb'),
      basename = 'mongodb-'+platform(true)+'-' + arch(true)+'-'+config.mongodb,
      pkg = {
        name: 'mongodb',
        version: config.mongodb,
        artifact: basename+'.tgz',
        checksum: '',
        url: 'http://fastdl.mongodb.org/'+platform(true)+'/'+basename+'.tgz'
      };

    async.series([download(pkg), extract(pkg)], function(err, res){
      if(err) return fn(err);

      var bin = path.resolve(res[1] + '/bin');
      process.env.PATH = bin + ':' + process.env.PATH;
      debug('add PATH ' + bin);
      fn();
    });
  };
}

function extract(pkg){
  return function(fn){
    var debug = require('debug')('mongoscope-ci:' + pkg.name +':extract'),
      tarball = path.resolve(config.cache + '/.artifacts/' + pkg.artifact),
      base = path.resolve(config.cache + '/' + pkg.name),
      dest = path.resolve(base + '/' + pkg.version),
      cleanup = createCleanupCrew('remove incomplete untar', function(){
      fs.removeSync(dest);
      debug('removed', dest);
    });

    fs.exists(dest, function(exists){
      if(exists){
        debug('already extracted', dest);
        cleanup.clear();
        fn(null, dest);
      }
      debug('reading ' + tarball);

      var input = fs.createReadStream(tarball),
        ungzip = zlib.createGunzip(),
        extractor = tar.Extract({path: dest, strip: 1});

      extractor.on('end', function(){
        debug('created', dest);
        cleanup.clear();
        fn(null, dest);
      });

      input.pipe(ungzip);
      ungzip.on('error', function(err){
        debug('error ungziping', err);
        cleanup.clear();
        fn(err, dest);
      }).on('finished', function(){
        debug('ungzipped');
      });

      debug('untar-ing....');
      ungzip.pipe(extractor);
    });
  };
}

function createCleanupCrew(msg, fn){
  var done = function(){
    console.log('process exit!');
    fn();
  };
  process.on('exit', done);
  return {
    clear: function(){
      process.removeListener('exit', done);
    }
  };
}
<commit_msg>bug: wasnt returning early if already extracted<commit_after>var async = require('async'),
  debug = require('debug')('mongoscope-ci'),
  request = require('request'),
  os = require('os'),
  path = require('path'),
  untildify = require('untildify'),
  fs = require('fs-extra'),
  zlib = require('zlib'),
  tar = require('tar'),
  spawn = require('child_process').spawn;

function platform(mongodb){
  var p = os.platform();
  if(mongodb && p === 'darwin') return 'osx';
  return p;
}

function arch(mongodb){
  var a = os.arch();
  if(mongodb && a === 'x64') return 'x86_64';
  return a;
}

function nodejs(){
  return function(fn){
    var debug = require('debug')('mongoscope-ci:nodejs'),
      basename = 'node-v'+config.nodejs+'-'+platform()+'-' + arch(),
      pkg = {
        name: 'nodejs',
        version: config.nodejs,
        artifact: basename+'.tar.gz',
        checksum: '',
        url: 'http://nodejs.org/dist/v'+config.nodejs+'/'+basename+'.tar.gz'
      };

    async.series([download(pkg), extract(pkg)], function(err, res){
      if(err) return fn(err);

      var bin = path.resolve(res[1] + '/bin');
      process.env.PATH = bin + ':' + process.env.PATH;
      debug('add PATH ' + bin);
      fn();
    });
  };
}

function mongodb(){
  return function(fn){
    var debug = require('debug')('mongoscope-ci:mongodb'),
      basename = 'mongodb-'+platform(true)+'-' + arch(true)+'-'+config.mongodb,
      pkg = {
        name: 'mongodb',
        version: config.mongodb,
        artifact: basename+'.tgz',
        checksum: '',
        url: 'http://fastdl.mongodb.org/'+platform(true)+'/'+basename+'.tgz'
      };

    async.series([download(pkg), extract(pkg)], function(err, res){
      if(err) return fn(err);

      var bin = path.resolve(res[1] + '/bin');
      process.env.PATH = bin + ':' + process.env.PATH;
      debug('add PATH ' + bin);
      fn();
    });
  };
}

function extract(pkg){
  return function(fn){
    var debug = require('debug')('mongoscope-ci:' + pkg.name +':extract'),
      tarball = path.resolve(config.cache + '/.artifacts/' + pkg.artifact),
      base = path.resolve(config.cache + '/' + pkg.name),
      dest = path.resolve(base + '/' + pkg.version),
      cleanup = createCleanupCrew('remove incomplete untar', function(){
      fs.removeSync(dest);
      debug('removed', dest);
    });

    fs.exists(dest, function(exists){
      if(exists){
        debug('already extracted', dest);
        cleanup.clear();
        return fn(null, dest);
      }
      debug('reading ' + tarball);

      var input = fs.createReadStream(tarball),
        ungzip = zlib.createGunzip(),
        extractor = tar.Extract({path: dest, strip: 1});

      extractor.on('end', function(){
        ended = true;
        debug('created', dest, arguments);
        cleanup.clear();
        fn(null, dest);
      });

      input.pipe(ungzip);
      ungzip.on('finished', function(){
        debug('ungzipped');
      });

      debug('untar-ing....');
      ungzip.pipe(extractor);
    });
  };
}

function createCleanupCrew(msg, fn){
  var done = function(){
    console.log('process exit!');
    fn();
  };
  process.on('exit', done);
  return {
    clear: function(){
      process.removeListener('exit', done);
    }
  };
}

{'fraction_non_alphanumeric': 0.19564886657538413, 'fraction_numerical': 0.0027384755819260614, 'mean_word_length': 2.748004561003421, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1928006', 'n_tokens_mistral': 2221, 'n_tokens_neox': 2120, 'n_words': 414}