<commit_before>/**
 * Module dependencies
 */

var fs = require('fs'),
    path = require('path'),
    utils = exports = module.exports;

/**
 * Does path `p` exist?
 *
 * @param {String} p
 * @return {Boolean}
 */

var exists = utils.exists = function(p) {
  return fs.existsSync(p);
};

/**
 * Returns `true` if `p` is a directory
 *
 * @param {String} p
 * @return {Boolean}
 */

var isDir = utils.isDir = function(p) {
  return exists(p) && fs.lstatSync(p).isDirectory();
};

/**
 * Returns `true` if `p` is a file
 *
 * @param {String} p
 * @return {Boolean}
 */

var isFile = utils.isFile = function(p) {
  return exists(p) && fs.lstatSync(p).isFile();
};

/**
 * Chunk `p` into parts
 *
 * @param {String} p
 * @return {Array}
 */

var splitPaths = utils.splitPaths = function(p) {
  return p.split(path.sep);
};

/**
 * Make all directories leading up to `p`
 * like a unix `mkdir -p p`
 *
 * @param {String} p
 * @return {Boolean}
 */

var mkdir = utils.mkdir = function(p) {
  var errors;

  splitPaths(p).reduce(function(dirs, dir) {
    dirs += dir + path.sep;
    var s = path.resolve(dirs);

    if(!exists(s)) {
      try {
        fs.mkdirSync(s);
      } catch(e) {
        errors = true;
      }
    }

    return dirs;
  }, '');

  return !errors;
};

/**
 * Read the path `p` and return an object with files and directories
 *
 * @param {String} p
 * @return {Object}
 */

var readDir = utils.readDir = function(p) {
  if(!isDir(p)) return false;
  var dir = fs.readdirSync(p).map(function(f) {
    return path.join(p, f);
  });

  return {
    files: dir.filter(function(f) {
      return isFile(f);
    }),
    directories: dir.filter(function(f) {
      return isDir(f);
    })
  };
};

/**
 * Read file `p`
 *
 * @param {String} p
 * @return {String|Boolean}
 */

var read = utils.read = function(p) {
  if(!exists(p)) return false;

  try {
    return fs.readFileSync(p);
  } catch(e) {
    return false;
  }
};

/**
 * Write `data` to path `p` creating any directories on the way.
 *
 * @param {String} p
 * @param {String} data
 * @return {Boolean}
 */

var write = utils.write = function(p, data) {
  mkdir(path.dirname(p));

  try {
    fs.writeFileSync(p, data);
    return true;
  } catch(e) {
    return false;
  }
};<commit_msg>add some new utilities to simplify core<commit_after>/**
 * Module dependencies
 */

var fs = require('fs'),
    path = require('path'),
    utils = exports = module.exports;

/**
 * Does path `p` exist?
 *
 * @param {String} p
 * @return {Boolean}
 */

var exists = utils.exists = function(p) {
  return fs.existsSync(p);
};

/**
 * Returns `true` if `p` is a directory
 *
 * @param {String} p
 * @return {Boolean}
 */

var isDir = utils.isDir = function(p) {
  return exists(p) && fs.lstatSync(p).isDirectory();
};

/**
 * Returns `true` if `p` is a file
 *
 * @param {String} p
 * @return {Boolean}
 */

var isFile = utils.isFile = function(p) {
  return exists(p) && fs.lstatSync(p).isFile();
};

/**
 * Chunk `p` into parts
 *
 * @param {String} p
 * @return {Array}
 */

var splitPaths = utils.splitPaths = function(p) {
  return p.split(path.sep);
};

/**
 * Return the name of file `p`
 *
 * @param {String} p
 * @return {String}
 */

var fileName = utils.fileName = function(p) {
  return path.basename(p, path.extname(p));
};

/**
 * Joins paths `a` and `b`, but replaces `b` ext with html
 *
 * @param {String} a
 * @param {String} b
 * @return {String}
 */

var joinPaths = utils.joinPaths = function(a, b) {
  var f = fileName(b),
      b = path.dirname(b);

  return path.join(a, b, f + '.html');
};

/**
 * Make all directories leading up to `p`
 * like a unix `mkdir -p p`
 *
 * @param {String} p
 * @return {Boolean}
 */

var mkdir = utils.mkdir = function(p) {
  var errors;

  splitPaths(p).reduce(function(dirs, dir) {
    dirs += dir + path.sep;
    var s = path.resolve(dirs);

    if(!exists(s)) {
      try {
        fs.mkdirSync(s);
      } catch(e) {
        errors = true;
      }
    }

    return dirs;
  }, '');

  return !errors;
};

/**
 * Read the path `p` and return an object with files and directories
 *
 * @param {String} p
 * @return {Object}
 */

var readDir = utils.readDir = function(p) {
  if(!isDir(p)) return false;
  var dir = fs.readdirSync(p).map(function(f) {
    return path.join(p, f);
  });

  return {
    files: dir.filter(function(f) {
      return isFile(f);
    }),
    directories: dir.filter(function(f) {
      return isDir(f);
    })
  };
};

/**
 * Read file `p`
 *
 * @param {String} p
 * @return {String|Boolean}
 */

var read = utils.read = function(p) {
  if(!exists(p)) return false;

  try {
    return fs.readFileSync(p, 'utf8');
  } catch(e) {
    return false;
  }
};

/**
 * Write `data` to path `p` creating any directories on the way.
 *
 * @param {String} p
 * @param {String} data
 * @return {Boolean}
 */

var write = utils.write = function(p, data) {
  mkdir(path.dirname(p));

  try {
    fs.writeFileSync(p, data);
    return true;
  } catch(e) {
    return false;
  }
};
{'fraction_non_alphanumeric': 0.1799680766161213, 'fraction_numerical': 0.00019952114924181964, 'mean_word_length': 2.7606901725431356, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6558030', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1859, 'n_words': 487}