<commit_before>var slug     = require('slug');
var fs       = require('fs');
var path     = require('path');
var Future   = require('data.future');
var Maybe    = require('data.maybe');
var utils    = require('./utils');
var FS       = require('io.filesystem')(fs);
var filterM  = require('control.monads').filterM;
var parallel = require('control.async')(Future).parallel;
var unary    = require('core.arity').unary;
var flip     = require('core.lambda').flip;
var zipWith  = require('data.array/zips/zip-with');
var extend   = require('xtend');

module.exports = function(homePath) {

  var novelHome      = λ[homePath];
  var joinPath       = λ a b -> path.join(a, b);
  var appendPath     = flip(joinPath);
  var extendWithPath = λ(data, path) -> extend(data, { path: path });

  function normaliseNovel(a) {
    if (a.title == null)
      throw new Error('<title> is required. ' + JSON.stringify(a));

    return {
      title      : a.title,
      author     : Maybe.fromNullable(a.author),
      modifiedAt : a.modifiedAt? Maybe.of(new Date(a.modifiedAt)) : Maybe.Nothing(),
      tags       : a.tags || []
    }
  }

  function novelList(dir) {
    return $do {
      folders <- FS.listDirectory(dir);
      dirs    <- filterM(Future, FS.isDirectory, folders.map(joinPath(dir)));
      files   <- parallel(dirs.map(appendPath('novel.json') ->> FS.readAsText));<commit_msg>Uses destructuring.<commit_after>var slug     = require('slug');
var fs       = require('fs');
var path     = require('path');
var Future   = require('data.future');
var Maybe    = require('data.maybe');
var extend   = require('xtend');
var FS       = require('io.filesystem')(fs);
var utils    = require('./utils');

var { filterM }  = require('control.monads');
var { parallel } = require('control.async')(Future);
var { unary }    = require('core.arity');
var { flip }     = require('core.lambda');
var zipWith      = require('data.array/zips/zip-with');

module.exports = function(homePath) {

  var novelHome      = λ[homePath];
  var joinPath       = λ a b -> path.join(a, b);
  var appendPath     = flip(joinPath);
  var extendWithPath = λ(data, path) -> extend(data, { path: path });

  function normaliseNovel(a) {
    if (a.title == null)
      throw new Error('<title> is required. ' + JSON.stringify(a));

    return {
      title      : a.title,
      author     : Maybe.fromNullable(a.author),
      modifiedAt : a.modifiedAt? Maybe.of(new Date(a.modifiedAt)) : Maybe.Nothing(),
      tags       : a.tags || []
    }
  }

  function novelList(dir) {
    return $do {
      folders <- FS.listDirectory(dir);
      dirs    <- filterM(Future, FS.isDirectory, folders.map(joinPath(dir)));
      files   <- parallel(dirs.map(appendPath('novel.json') ->> FS.readAsText));
{'fraction_non_alphanumeric': 0.16781442551649148, 'fraction_numerical': 0.0, 'mean_word_length': 3.011627906976744, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6505900', 'n_tokens_mistral': 910, 'n_tokens_neox': 878, 'n_words': 194}