<commit_before>
module.exports = function (options) { // wrapper in case we're in module_context mode

"use strict";

var cache   = new (require('inmemfilecache'));
var Feed    = require('feed');
var fs      = require('fs');
var glob    = require('glob');
var marked  = require('marked');
var path    = require('path');
var Promise = require('Promise');
var sitemap = require('sitemap');
var subst   = new (require('./subst'));
var utils   = require('./../../lib/utils');

var executeP = Promise.denodeify(utils.execute);

marked.setOptions({
  rawHtml: true,
  //pedantic: true,
});


subst.registerReplaceHandler('include', function(filename, params) {
  return subst.replaceParams(readFile(filename, {encoding: "utf-8"}), params);
});

subst.registerReplaceHandler('example', function(options) {

  options.width = options.width || "400";
  options.height = options.height || "300";

  return subst.replaceParams(readFile("dev/templates/example.template"), options);
});

subst.registerReplaceHandler('diagram', function(options) {

  options.width = options.width || "400";
  options.height = options.height || "300";

  return subst.replaceParams(readFile("dev/templates/diagram.template"), options);
});

var readFile = function(fileName) {
  return cache.readFileSync(fileName, "utf-8");
};

var Builder = function(options) {

  var g_articles = [];

  function joinUrl() {
    return Array.prototype.map(function(part) {
      return (part.substr(-1) === "/") ? part.substr(0, part.length - 1) : part;
    }).join("/");
  }

  function writeFileIfChanged(fileName, content) {
    if (fs.existsSync(fileName)) {
      var old = readFile(fileName);
      if (content == old) {
        return;
      }
    }
    fs.writeFileSync(fileName, content);
    console.log("Wrote: " + fileName);
  };

  var extractHeader = (function() {
    var headerRE = /([A-Z0-9_-]+): (.*?)$/i;

    return function(content) {
      var metaData = { };
      var lines = content.split("\n");
      while (true) {
        var line = lines[0].trim();
        var m = headerRE.exec(line);
        if (!m) {
          break;
        }
        metaData[m[1].toLowerCase()] = m[2];
        lines.shift();
      }
      return {
        content: lines.join("\n"),
        headers: metaData,
      };
    };
  }());

  var loadMD = function(contentFileName) {
    var content = cache.readFileSync(contentFileName, "utf-8");
    return extractHeader(content);
  };

  var applyTemplateToFile = function(templatePath, contentFileName, outFileName, opt_extra) {
    console.log("processing: ", contentFileName);
    var template = readFile(templatePath);
    var data = loadMD(contentFileName);<commit_msg>make it possible to set template for docs<commit_after>
module.exports = function (options) { // wrapper in case we're in module_context mode

"use strict";

var cache   = new (require('inmemfilecache'));
var Feed    = require('feed');
var fs      = require('fs');
var glob    = require('glob');
var marked  = require('marked');
var path    = require('path');
var Promise = require('Promise');
var sitemap = require('sitemap');
var subst   = new (require('./subst'));
var utils   = require('./../../lib/utils');

var executeP = Promise.denodeify(utils.execute);

marked.setOptions({
  rawHtml: true,
  //pedantic: true,
});


subst.registerReplaceHandler('include', function(filename, params) {
  return subst.replaceParams(readFile(filename, {encoding: "utf-8"}), params);
});

subst.registerReplaceHandler('example', function(options) {

  options.width = options.width || "400";
  options.height = options.height || "300";

  return subst.replaceParams(readFile("dev/templates/example.template"), options);
});

subst.registerReplaceHandler('diagram', function(options) {

  options.width = options.width || "400";
  options.height = options.height || "300";

  return subst.replaceParams(readFile("dev/templates/diagram.template"), options);
});

var readFile = function(fileName) {
  return cache.readFileSync(fileName, "utf-8");
};

var Builder = function(options) {

  var g_articles = [];

  function joinUrl() {
    return Array.prototype.map(function(part) {
      return (part.substr(-1) === "/") ? part.substr(0, part.length - 1) : part;
    }).join("/");
  }

  function writeFileIfChanged(fileName, content) {
    if (fs.existsSync(fileName)) {
      var old = readFile(fileName);
      if (content == old) {
        return;
      }
    }
    fs.writeFileSync(fileName, content);
    console.log("Wrote: " + fileName);
  };

  var extractHeader = (function() {
    var headerRE = /([A-Z0-9_-]+): (.*?)$/i;

    return function(content) {
      var metaData = { };
      var lines = content.split("\n");
      while (true) {
        var line = lines[0].trim();
        var m = headerRE.exec(line);
        if (!m) {
          break;
        }
        metaData[m[1].toLowerCase()] = m[2];
        lines.shift();
      }
      return {
        content: lines.join("\n"),
        headers: metaData,
      };
    };
  }());

  var loadMD = function(contentFileName) {
    var content = cache.readFileSync(contentFileName, "utf-8");
    return extractHeader(content);
  };

  var applyTemplateToFile = function(defaultTemplatePath, contentFileName, outFileName, opt_extra) {
    console.log("processing: ", contentFileName);
    var data = loadMD(contentFileName);
    var templatePath = data.headers.template || defaultTemplatePath;
    var template = readFile(templatePath);
{'fraction_non_alphanumeric': 0.16471449487554904, 'fraction_numerical': 0.00841874084919473, 'mean_word_length': 3.407258064516129, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1016173', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1660, 'n_words': 364}