<commit_before>const fs = require('fs');
const isEmptyObject = require('is-empty-object');
const mkdirp = require('mkdirp').sync;
const path = require('path');
const replaceExt = require('replace-ext');
const statusLog = require('./util/statusLog');
const through = require('through2');
const vfs = require('vinyl-fs');

/**
 * Options that can be passed to `Spacedoc.init()`.
 * @typedef {Object} InitOptions
 * @prop {Boolean} [incremental=false] Enable incremental compiling. On subsequent runs of `Spacedoc.init()` (such as in a Gulp task, after a file has been saved), the plugin will not reset the cached list of pages.
 */

/**
 * Initialize Spacedoc, parsing a set of documentation pages and outputting HTML files. To call this function, `Spacedoc.config()` must have been called previously with the `template` parameter set.
 * This function can be called standalone, with the `src` and `dest` options having been set in `Spacedoc.config()`, or it can be called in a Gulp stream, and `src` and `dest` can be omitted.
 *
 * @example <caption>Use within a Gulp stream:</caption>
 *   gulp.src('docs/*.md')
 *     .pipe(Spacedoc.init())
 *     .pipe(gulp.dest('dist'));
 *
 * @example <caption>Use standalone:</caption>
 *   Spacedoc.init({
 *     src: 'docs/*.md',
 *     dest: 'dist'
 *   })
 *     .on('finish', () => {)
 *       // Parsing finished
 *     });
 *
 * @param {InitOptions} opts - Options for initialization.
 * @returns {Stream.Writable.<Vinyl>} A stream containing the modified files. You can add `on('finish')` after `Spacedoc.init()` to listen for when processing is done.
 */
module.exports = function init(opts = {}) {
  // Initialize options if Spacedoc.config() was not called
  if (isEmptyObject(this.options)) {
    this.config();
  }

  // Reset the internal data tree
  if (!opts.incremental) {
    this.tree = [];
  }

  // Make the destination folder
  if (this.options.dest) {
    mkdirp(this.options.dest);
  }

  // If `src` was passed, make an ad-hoc stream for processing
  if (this.options.src) {
    return vfs
      .src(this.options.src, { base: this.options.base })
      .pipe(transform.apply(this))
  }
  // Otherwise, go straight to the transform function.
  else {
    const stream = transform.apply(this);
    return stream;
  }

  /**
   * Create a stream transform function to manipulate files.
   * @private
   * @returns {Function} Transform function.
   */
  function transform() {
    /**
     * Stream transform function. Takes in a Vinyl file for a documentation page and makes these changes:
     *   - Parses Front Matter for Spacedoc adapters and collects documentation data
     *   - Converts Markdown to HTML (unless configured to ignore it)
     *   - Changes the extension to `.html` (or whatever was configured)
     *   - Removes Front Matter (unless configured to keep it)
     *   - Replaces the page contents with the loaded template
     *
     * @private
     * @param {Vinyl} file - File to transform.
     * @param {String} enc - File encoding.
     * @param {Function} cb - Callback to return the modified file.
     */
    return through.obj((file, enc, cb) => {
      const time = process.hrtime();

      this.parse(file, opts).then(data => {
        // Change the extension of the incoming file, and replace the Markdown contents with rendered HTML
        file.path = replaceExt(file.path, `.${this.options.extension}`);
        file.contents = new Buffer(this.build(data));

        // Write new file to disk if necessary
        if (this.options.dest) {
          const filePath = path.join(this.options.dest, path.basename(file.path));
          fs.writeFileSync(filePath, file.contents.toString());
        }

        // Log page name, processing time, and adapters used to console
        if (!this.options.silent) {
          statusLog(path.basename(file.path), data, process.hrtime(time));
        }

        cb(null, file);
      });
    });
  };
}<commit_msg>In init(), defer generating HTML until every Markdown file has been processed. This opens up the ability to render content aware of site structure in pages<commit_after>const File = require('vinyl');
const fs = require('fs');
const isEmptyObject = require('is-empty-object');
const mkdirp = require('mkdirp').sync;
const path = require('path');
const replaceExt = require('replace-ext');
const statusLog = require('./util/statusLog');
const through = require('through2');
const vfs = require('vinyl-fs');

/**
 * Options that can be passed to `Spacedoc.init()`.
 * @typedef {Object} InitOptions
 * @prop {Boolean} [incremental=false] Enable incremental compiling. On subsequent runs of `Spacedoc.init()` (such as in a Gulp task, after a file has been saved), the plugin will not reset the cached list of pages.
 */

/**
 * Initialize Spacedoc, parsing a set of documentation pages and outputting HTML files. To call this function, `Spacedoc.config()` must have been called previously with the `template` parameter set.
 * This function can be called standalone, with the `src` and `dest` options having been set in `Spacedoc.config()`, or it can be called in a Gulp stream, and `src` and `dest` can be omitted.
 *
 * @example <caption>Use within a Gulp stream:</caption>
 *   gulp.src('docs/*.md')
 *     .pipe(Spacedoc.init())
 *     .pipe(gulp.dest('dist'));
 *
 * @example <caption>Use standalone:</caption>
 *   Spacedoc.init({
 *     src: 'docs/*.md',
 *     dest: 'dist'
 *   })
 *     .on('finish', () => {)
 *       // Parsing finished
 *     });
 *
 * @param {InitOptions} opts - Options for initialization.
 * @returns {Stream.Writable.<Vinyl>} A stream containing the modified files. You can add `on('finish')` after `Spacedoc.init()` to listen for when processing is done.
 */
module.exports = function init(opts = {}) {
  // Initialize options if Spacedoc.config() was not called
  if (isEmptyObject(this.options)) {
    this.config();
  }

  // Necessary because through2's end stream function below can't be an arrow function
  const _this = this;

  // Reset the internal data tree
  if (!opts.incremental) {
    this.tree = [];
  }

  // Make the destination folder
  if (this.options.dest) {
    mkdirp(this.options.dest);
  }

  // If `src` was passed, make an ad-hoc stream for processing
  if (this.options.src) {
    return vfs
      .src(this.options.src, { base: this.options.base })
      .pipe(transform.apply(this))
  }
  // Otherwise, go straight to the transform function.
  else {
    return transform.apply(this);
  }

  /**
   * Create a stream transform function to manipulate files.
   * @private
   * @returns {Function} Transform function.
   */
  function transform() {
    return through.obj(
      /**
       * Stream transform function. Generates page data from a file and stores it in Spacedoc's page list. This function does not return a modified file to the stream, because the files aren't written until every page has been processed.
       * @private
       * @param {Vinyl} file - File to transform.
       * @param {String} enc - File encoding.
       * @param {Function} cb - Callback to return the modified file.
       */
      (file, enc, cb) => {
        const time = process.hrtime();

        this.parse(file, opts).then(() => cb());
      },
      /**
       * Stream flush function. Iterates through the created page data and renders HTML files for each one.
       * @private
       * @this stream.Transform
       * @param {Function} cb - Callback that signals the function is finished.
       */
      function(cb) {
        _this.tree.map(page => {
          const file = new File({
            path: replaceExt(page.fileName, `.${_this.options.extension}`),
            cwd: _this.options.dest || _this.options.pageRoot,
            contents: new Buffer(_this.build(page)),
          });

          // Write new file to disk if necessary
          if (_this.options.dest) {
            const filePath = path.join(_this.options.dest, path.basename(file.path));
            fs.writeFileSync(filePath, file.contents.toString());
          }

          // Push finished file through stream
          this.push(file);

          // Log page name, processing time, and adapters used to console
          if (!_this.options.silent) {
            statusLog(path.basename(file.path), page, process.hrtime(time));
          }
        });

        cb();
      }
    );
  }
}
{'fraction_non_alphanumeric': 0.12679083094555874, 'fraction_numerical': 0.00035816618911174784, 'mean_word_length': 2.8568139963167587, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4838239', 'n_tokens_mistral': 2433, 'n_tokens_neox': 2362, 'n_words': 830}