<commit_before>const _ = require('./lib/lodash-extended');
const argv = require('yargs').argv;
const fs = require('fs-extra');
const path = require('path');
const http = require('http');
const open = require('open');
const ecstatic = require('ecstatic');
const contentful = require('contentful');

require('app-module-path').addPath(path.join(__dirname, 'lib'));
const requireDir = require('require-dir');

const lib = requireDir('./lib', {camelcase: true});
const plugins = requireDir('./lib/plugins', {camelcase: true});
const config = require('./config');

const mill = {parseContent, pages, clean, make, build, dev, preview, serve};
const cmd = argv._[0];

module.exports = runMill;

function runMill(cmd) {
  cmd = cmd || config.defaultCommand;
  mill[cmd]();
}

function parseContent(...args) {
  return lib.parseContent(...args);
}

function pages(...args) {
  return lib.pages(...args);
}

function clean() {
  fs.removeSync(config.destBase);
}

function dev() {
  make().then(mill.serve);
}

function preview() {
  build().then(mill.serve);
}

function build() {
  return make();
}

function make() {
  mill.clean();

  const pathsSource = 'src/wrapper.json';
  const assetGroupPaths = _.mapValues(fs.readJsonSync(pathsSource), paths => {
    return _.map(paths, _path => path.normalize(path.join(path.dirname(pathsSource), _path)));
  });

  return _(assetGroupPaths)
    .thru(normalize)
    .mapValues(generateAssets)
    .resolveAsyncObject()
    .value()
    .then(result => getViews(config.contentful, result));

  function getViews(keys, assetPaths) {
    return keys ? getContent(keys, assetPaths) : mill.pages(assetPaths);
  }

  function getContent(keys, assetPaths) {
    return contentful.createClient(keys).getEntries().then(entries => {
      return mill.pages(_.assign(assetPaths, mill.parseContent(entries.items)));
    });
  }
}

function normalize(assetGroupPaths) {
  return _(assetGroupPaths)
    .mapValues(lib.normalizeToObjects)
    .mapValues(lib.normalizePaths)
    .mapValues(lib.normalizePathPipelines)
    .mapValues(lib.normalizeGroups)
    .value();
}

function generateAssets(group) {
  return _(group.files)
    .mapWhenElse('isFile', plugins.read, (val) => Promise.resolve(val))
    .mapAsyncWhen(['isFile', 'shouldCompile'], plugins.compile)
    .mapAsyncWhen(['isFile', 'shouldPostProcess'], plugins.postProcess)
    .mapAsyncWhenElse('content', plugins.output, plugins.copy)
    .mapAsyncWhenFilter('webPath', plugins.toWebPath)
    .thru(val => Promise.all(val))
    .value();
}

function serve() {
  http.createServer(ecstatic(config.serveRoot)).listen(config.servePort);
  console.log(config.serveMsg);
  open(config.servePath);
}<commit_msg>move some functions out of make task<commit_after>const _ = require('./lib/lodash-extended');
const argv = require('yargs').argv;
const fs = require('fs-extra');
const path = require('path');
const http = require('http');
const open = require('open');
const ecstatic = require('ecstatic');
const contentful = require('contentful');

require('app-module-path').addPath(path.join(__dirname, 'lib'));
const requireDir = require('require-dir');

const lib = requireDir('./lib', {camelcase: true});
const plugins = requireDir('./lib/plugins', {camelcase: true});
const config = require('./config');

const mill = {parseContent, pages, clean, make, build, dev, preview, serve};
const cmd = argv._[0];

module.exports = runMill;

function runMill(cmd) {
  cmd = cmd || config.defaultCommand;
  mill[cmd]();
}

function parseContent(...args) {
  return lib.parseContent(...args);
}

function pages(...args) {
  return lib.pages(...args);
}

function clean() {
  fs.removeSync(config.destBase);
}

function dev() {
  make().then(mill.serve);
}

function preview() {
  build().then(mill.serve);
}

function build() {
  return make();
}

function make() {
  mill.clean();

  const pathsSource = 'src/wrapper.json';
  const assetGroupPaths = _.mapValues(fs.readJsonSync(pathsSource), paths => {
    return _.map(paths, _path => path.normalize(path.join(path.dirname(pathsSource), _path)));
  });

  return _(assetGroupPaths)
    .thru(normalize)
    .mapValues(generateAssets)
    .resolveAsyncObject()
    .value()
    .then(result => getViews(config.contentful, result));
}

function normalize(assetGroupPaths) {
  return _(assetGroupPaths)
    .mapValues(lib.normalizeToObjects)
    .mapValues(lib.normalizePaths)
    .mapValues(lib.normalizePathPipelines)
    .mapValues(lib.normalizeGroups)
    .value();
}

function generateAssets(group) {
  return _(group.files)
    .mapWhenElse('isFile', plugins.read, (val) => Promise.resolve(val))
    .mapAsyncWhen(['isFile', 'shouldCompile'], plugins.compile)
    .mapAsyncWhen(['isFile', 'shouldPostProcess'], plugins.postProcess)
    .mapAsyncWhenElse('content', plugins.output, plugins.copy)
    .mapAsyncWhenFilter('webPath', plugins.toWebPath)
    .thru(val => Promise.all(val))
    .value();
}

function getViews(keys, assetPaths) {
  return keys ? getContent(keys, assetPaths) : mill.pages(assetPaths);
}

function getContent(keys, assetPaths) {
  return contentful.createClient(keys).getEntries().then(entries => {
    return mill.pages(_.assign(assetPaths, mill.parseContent(entries.items)));
  });
}

function serve() {
  http.createServer(ecstatic(config.serveRoot)).listen(config.servePort);
  console.log(config.serveMsg);
  open(config.servePath);
}
{'fraction_non_alphanumeric': 0.16669760534620381, 'fraction_numerical': 0.00037126415444588826, 'mean_word_length': 4.850162866449511, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '386697', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1685, 'n_words': 324}