<commit_before>//
//  fs2.js
//

var fs2 = exports;

var fs = require('fs'),
    path = require('path'),
    mkdirp = require('mkdirp'),
    traverse = require('traverse'),
    findit = require('findit');

//
// Transform array into flat object
// Uses as a helper.
//
fs2.filesArrayToObject = function(files, resolve){
  var _files = {};
  files.forEach(function(file){
    var contents = '';
    if (resolve) {
      //
      // Remark: also read the contents of the file
      // and set it in memory on JSON object as binary data
      //
      var stat = fs.statSync(file);

      if (stat.isDirectory()) {
        contents = { "isDirectory": true };
      } else {
        contents = fs.readFileSync(file);
      }

    }
    _files[file] = contents;
  });
  return _files;
}

//
// Convert flat object of file paths into nested JSON object
//
fs2.filesToTree = function(files){
  
  files = files.reduce(function (acc, file) { 
    var ps = file.split('/');
    ps = ps.filter(function (e) {
      var isdot = (e === ".");
      var isUndef = (e === "undefined");
      if (isUndef) { console.log("undefined from "+file); }
      return !isdot && !isUndef;
    });

    ps.splice(0,1);
    traverse(acc).set(ps, {});
    return acc;
  }, {});

  // why is this happening?
  delete files.undefined;

  return files;
  
}


//
// Recursively read directory into flat JSON object
// arguments: path, resolve, filter
//
fs2.readDir = function (p, resolve, filter, cb) {<commit_msg>[api] A modified ToTree for ordered ToC<commit_after>//
//  fs2.js
//

var fs2 = exports;

var fs = require('fs'),
    path = require('path'),
    mkdirp = require('mkdirp'),
    traverse = require('traverse'),
    Hash = require('hashish'),
    findit = require('findit');

//
// Transform array into flat object
// Uses as a helper.
//
fs2.filesArrayToObject = function(files, resolve){
  var _files = {};
  files.forEach(function(file){
    var contents = '';
    if (resolve) {
      //
      // Remark: also read the contents of the file
      // and set it in memory on JSON object as binary data
      //
      var stat = fs.statSync(file);

      if (stat.isDirectory()) {
        contents = { "isDirectory": true };
      } else {
        contents = fs.readFileSync(file);
      }

    }
    _files[file] = contents;
  });
  return _files;
}

//
// Convert flat object of file paths into nested JSON object
//
fs2.filesToTree = function(files){
  
  files = files.reduce(function (acc, file) { 
    var ps = file.split('/');
    ps = ps.filter(function (e) {
      var isdot = (e === ".");
      var isUndef = (e === "undefined");
      if (isUndef) { console.log("undefined from "+file); }
      return !isdot && !isUndef;
    });

    ps.splice(0,1);
    traverse(acc).set(ps, {});
    return acc;
  }, {});

  // why is this happening?
  delete files.undefined;

  return files;
  
}


// I want to build up a directory tree to use for the table of contents.
fs2.dirToTree = function (dir) {

  //Very similar to filesToTree. Builds your basic nested deal.
  var _tree = Object.keys(dir).reduce(function (acc, p) {
    var ps = ("./"+path.normalize("./pages/"+p)).split("/");

    traverse(acc).set(ps, {});
    return acc;

  }, {});

  // we need some array in order to impose an ordering.
  var ordered = function ordered (tree) {
    var hash = Hash(tree).map(function (v) {
      var files = [];
      Object.keys(v).forEach( function (sv) {
        // order-ify the next layer down
        ordered(v[sv]).forEach(function(o) {
          files.push(o);
        });
      });
      return files;
    }).items;

    var _tree = [];
    Object.keys(hash).forEach(function(k) {
      var obj = {};
      obj[k] = hash[k];
      _tree.push(obj);
    });

    return _tree;
  }

  _tree = ordered(_tree);

  // rebuilding the filepath
  _tree = traverse(_tree).map( function (e) {
    if (this.isLeaf) {
      var p = this.path.filter(function (i) {
        return isNaN(Number(i));
      }).join("/");
      if (dir[p]) {
        return dir[p];
      } else {
        return p;
      }
    }
    else {
      return e;
    }
  });


  return _tree;
}


//
// Recursively read directory into flat JSON object
// arguments: path, resolve, filter
//
fs2.readDir = function (p, resolve, filter, cb) {
{'fraction_non_alphanumeric': 0.15959925442684064, 'fraction_numerical': 0.003494874184529357, 'mean_word_length': 2.2896551724137932, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4953887', 'n_tokens_mistral': 1423, 'n_tokens_neox': 1385, 'n_words': 390}