<commit_before>var fs = require('vinyl-fs');
var path = require('path');
var Promise = require('bluebird');
var collect = Promise.promisify(require('collect-stream'));
var writeFile = Promise.promisify(require('fs-extra').outputFile);
var del = Promise.promisify(require('del'));
var l = require('lodash');

var S = require('./lib');

function getData(src) {
  return fs.src(src)
  .pipe(S.loadCson())
  .pipe(S.loadJson())
  .pipe(S.loadCsonFrontmatter())
  .pipe(S.loadMarkdown())

  ;
}

function getSchemas(src) {
  return fs.src(src)
  .pipe(S.loadCson())
  .pipe(S.loadJson())
  ;
}

function getTemplates(src) {
  return fs.src(src)
  .pipe(S.loadCsonFrontmatter())
  ;
}

function buildSiliconZucchini(opts) {
  var settings = l.defaults({}, opts, {
    data: './data/**/*.{json,cson,md}',
    schemas: './src/schemas/**/*.{json,cson}',
    templates: './src/**/*.html',
    processData: l.identity,
    createRoutes: l.identity,
    destination: 'build',
    templateHelpers: {}
  });

  return Promise.all([
    collect(settings.processData(getData(settings.data))),
    collect(getSchemas(settings.schemas))
    .then(function (schemas) {
      return l.indexBy(l.pluck(schemas, 'data'), 'title');
    }),
    collect(getTemplates(settings.templates)),
    del('build')
  ])
  .tap(function (inputs) {
    inputs[0].map(function (item) {<commit_msg>How to Build a Silicon Zucchini<commit_after>var fs = require('vinyl-fs');
var path = require('path');
var Promise = require('bluebird');
var collect = Promise.promisify(require('collect-stream'));
var writeFile = Promise.promisify(require('fs-extra').outputFile);
var del = Promise.promisify(require('del'));
var l = require('lodash');

var S = require('./lib');

function getData(src) {
  return fs.src(src)
  .pipe(S.loadCson())
  .pipe(S.loadJson())
  .pipe(S.loadCsonFrontmatter())
  .pipe(S.loadMarkdown())
  ;
}

function getSchemas(src) {
  return fs.src(src)
  .pipe(S.loadCson())
  .pipe(S.loadJson())
  ;
}

function getTemplates(src) {
  return fs.src(src)
  .pipe(S.loadCsonFrontmatter())
  ;
}

/**
 * # Build a Silicon Zucchini
 * @param {Object}    opts               A truckload of options
 * @param {String}   [opts.data]         Path glob to find data files
 * @param {String}   [opts.schemas]      Path glob to find schema files
 * @param {String}   [opts.templates]    Path glob to find template files
 * @param {Function} [opts.processData]  Takes a vinyl-fs stream of data files,
 *   manipulates that data, returns a vinyl-fs stream
 * @param {Function} [opts.createRoutes] Callback to create a route object.
 *   Input parameters are `data: [{}], schemas: [{}], getTemplate: Function`.
 *   Needs to return an array of Route objects or arrays (it will be flattened).
 *   Route objects have the shape `{data: {}, route: String, layout: String}`.
 * @param {String}   [opts.destination='build']  Output folder.
 * @param {Object}   [opts.templateHelpers] Map of additonal function that will
 *   be available in templates.
 * @return {Promise} Will be resolved when all processing is done.
 */
function buildSiliconZucchini(opts) {
  var settings = l.defaults({}, opts, {
    data: './data/**/*.{json,cson,md}',
    schemas: './src/schemas/**/*.{json,cson}',
    templates: './src/**/*.html',
    processData: l.identity,
    createRoutes: l.identity,
    destination: 'build',
    templateHelpers: {}
  });

  return Promise.all([
    collect(settings.processData(getData(settings.data))),
    collect(getSchemas(settings.schemas))
    .then(function (schemas) {
      return l.indexBy(l.pluck(schemas, 'data'), 'title');
    }),
    collect(getTemplates(settings.templates)),
    del('build')
  ])
  .tap(function (inputs) {
    inputs[0].map(function (item) {
{'fraction_non_alphanumeric': 0.17111585691404163, 'fraction_numerical': 0.0005339028296849973, 'mean_word_length': 3.891644908616188, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '150505', 'n_tokens_mistral': 1263, 'n_tokens_neox': 1238, 'n_words': 284}