<commit_before>'use strict';
var git = require('gift');
var fs = require('fs');
var fstream = require('fstream');
var jsforce = require('jsforce');
var async = require('async');
var AdmZip = require('adm-zip');

//muts all logs
var MUTE = false;

/*
 * Creates the return object for the mainCallback
 */
function createReturnObject(err, msg){
    return {
        error: err,
        details: msg
    };
}

/*
 * Sync Deletes a folder recursively
 * @path: folder path
 * @exclude: exclude a certain folder's name
 * @doNotDeleteRoot: do not delete root folder
 */
var deleteFolderRecursive = function(path, excludeFolder, doNotDeleteRoot) {
  if( fs.existsSync(path) ) {
    fs.readdirSync(path).forEach(function(file,index){
      var curPath = path + "/" + file;
      if(fs.lstatSync(curPath).isDirectory()) { // recurse
        if(!excludeFolder || file != exclude){
            deleteFolderRecursive(curPath, exclude);
        }
      } else { // delete file
        fs.unlinkSync(curPath);
      }
    });
    if(!doNotDeleteRoot) fs.rmdirSync(path);
  }
};


module.exports = {
    doAll : function(mainCallback){

        //status object<commit_msg>issue #1<commit_after>'use strict';
var git = require('gift');
var fs = require('fs');
var fstream = require('fstream');
var jsforce = require('jsforce');
var async = require('async');
var AdmZip = require('adm-zip');

//muts all logs
var MUTE = false;

/*
 * Creates the return object for the mainCallback
 */
function createReturnObject(err, msg){
    return {
        error: err,
        details: msg
    };
}

/*
 * Sync Deletes a folder recursively
 * @path: folder path
 * @exclude: exclude a certain folder's name
 * @doNotDeleteRoot: do not delete root folder
 */
var deleteFolderRecursive = function(path, exclude, doNotDeleteRoot) {
  if( fs.existsSync(path) ) {
    fs.readdirSync(path).forEach(function(file,index){
      var curPath = path + "/" + file;
      if(fs.lstatSync(curPath).isDirectory()) { // recurse
        if(!exclude || file != exclude){
            deleteFolderRecursive(curPath, exclude);
        }
      } else { // delete file
        fs.unlinkSync(curPath);
      }
    });
    if(!doNotDeleteRoot) fs.rmdirSync(path);
  }
};


module.exports = {
    doAll : function(mainCallback){

        //status object
{'fraction_non_alphanumeric': 0.14049586776859505, 'fraction_numerical': 0.00043497172683775554, 'mean_word_length': 3.107142857142857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5020502', 'n_tokens_mistral': 723, 'n_tokens_neox': 706, 'n_words': 182}