<commit_before>var _ = require('lodash');
var cheerio = require('cheerio');
var Decimal = require('decimal.js');
var fs = require('fs');
var mkdirp = require('mkdirp');
var multiGlob = require('./util').multiGlob;
var nunjucks = require('nunjucks');
var path = require('path');
var Penrose = require('penrose').Penrose;
var Promise = require('bluebird');
var Remarkable = require('remarkable');
var toc = require('markdown-toc');
var yaml = require('js-yaml');

Promise.promisifyAll(fs);

var mkdirpAsync = Promise.promisify(mkdirp);

var nunjucksEnv = nunjucks.configure();

/**
 *
 * @param  {Object} config Keys: src, dist, plugins, imageStyles, penrose, env
 * @return {Toco}
 */
function Toco(config) {
  this.config = config;

  this.flags = {
    initMdParser: false // Whether markdown parser has been initialised
  };

  nunjucksEnv.addFilter('isNumber', function (value) {
    return _.isNumber(value);
  });

  nunjucksEnv.addFilter('kebabCase', function (value) {
    return _.kebabCase(value);
  });

  nunjucksEnv.addFilter('remove', function (html, selector) {
    var $ = cheerio.load(html);
    $(selector).remove();
    return $.html();
  });

  nunjucksEnv.addFilter('toDecimalPlaces', function (value, dp, rm) {
    var decimal = new Decimal(value);
    return decimal.toDecimalPlaces(dp, rm).valueOf();
  });

  // _.forEach(config.env.extensions, function (ext, name) {
  //   nunjucksEnv.addExtension(name, ext);
  // });

  var DataExtension = function () {
    this.tags = ['data'];

    this.parse = function (parser, nodes, lexer) {
      var tok = parser.nextToken();
      var args = parser.parseSignature(null, true);

      parser.advanceAfterBlockEnd(tok.value);
      parser.parseUntilBlocks('end' + tok.value);
      parser.advanceAfterBlockEnd();

      return new nodes.CallExtension(this, 'run', args);
    };

    this.run = function (context, name, dataSrc, filter) {
      var data = yaml.safeLoad(fs.readFileSync(config.src + '_data/' + dataSrc + '.yml', 'utf8'));

      if (filter) {
        data = _.filter(data, filter);
      }

      var str = nunjucks.render(config.src + '_includes/' + name + '.njk', {
        data: data
      });

      return new nunjucks.runtime.SafeString(str);
    };
  };

  nunjucksEnv.addExtension('data', new DataExtension);
}

Toco.prototype = {
  constructor: Toco,
  /**
   * Initialises markdown parser.
   */
  initMdParser: function () {
    if (this.flags.initMdParser) {
      return;
    }

    var md = new Remarkable({
      html: true
    });

    md.config = this.config;
<commit_msg>Add template and data file resolutions to data extension<commit_after>var _ = require('lodash');
var cheerio = require('cheerio');
var Decimal = require('decimal.js');
var fs = require('fs');
var mkdirp = require('mkdirp');
var multiGlob = require('./util').multiGlob;
var nunjucks = require('nunjucks');
var path = require('path');
var Penrose = require('penrose').Penrose;
var Promise = require('bluebird');
var Remarkable = require('remarkable');
var toc = require('markdown-toc');
var yaml = require('js-yaml');

Promise.promisifyAll(fs);

fs.existsAsync = Promise.promisify(function exists2(path, exists2callback) {
  fs.exists(path, function callbackWrapper(exists) {
    exists2callback(null, exists);
  });
});

var mkdirpAsync = Promise.promisify(mkdirp);

var nunjucksEnv = nunjucks.configure();

/**
 *
 * @param  {String} file File basename, file basename with extension, or path
 * @param  {String} dir
 * @param  {Array} extnames
 * @return {String}
 */
function resolveFileLocation(file, dir, extnames) {
  var i, il;

  if (file.substring(0, 1) === '/' || file.substring(0, 2) === './' || file.substring(0, 3) === '../') {
    if (fs.existsSync(file)) {
      return file;
    }

    for (i = 0, il = extnames.length; i < il; i++) {
      if (fs.existsSync(file + extnames[i])) {
        return file + extnames[i];
      }
    }

    return;
  }

  if (fs.existsSync(dir + file)) {
    return dir + file;
  }

  for (i = 0, il = extnames.length; i < il; i++) {
    if (fs.existsSync(dir + file + extnames[i])) {
      return dir + file + extnames[i];
    }
  }
}

/**
 *
 * @param  {Object} config Keys: src, dist, plugins, imageStyles, penrose, env
 * @return {Toco}
 */
function Toco(config) {
  this.config = config;

  this.flags = {
    initMdParser: false // Whether markdown parser has been initialised
  };

  nunjucksEnv.addFilter('isNumber', function (value) {
    return _.isNumber(value);
  });

  nunjucksEnv.addFilter('kebabCase', function (value) {
    return _.kebabCase(value);
  });

  nunjucksEnv.addFilter('remove', function (html, selector) {
    var $ = cheerio.load(html);
    $(selector).remove();
    return $.html();
  });

  nunjucksEnv.addFilter('toDecimalPlaces', function (value, dp, rm) {
    var decimal = new Decimal(value);
    return decimal.toDecimalPlaces(dp, rm).valueOf();
  });

  // _.forEach(config.env.extensions, function (ext, name) {
  //   nunjucksEnv.addExtension(name, ext);
  // });

  var DataExtension = function () {
    this.tags = ['data'];

    this.parse = function (parser, nodes, lexer) {
      var tok = parser.nextToken();
      var args = parser.parseSignature(null, true);

      parser.advanceAfterBlockEnd(tok.value);
      parser.parseUntilBlocks('end' + tok.value);
      parser.advanceAfterBlockEnd();

      return new nodes.CallExtension(this, 'run', args);
    };

    this.run = function (context, templateFile, dataFile, dataFilter) {
      var templateFileResolved = resolveFileLocation(templateFile, config.src + '_includes/', ['.njk']);
      var dataFileResolved = resolveFileLocation(dataFile, config.src + '_data/', ['.yml']);

      if (_.isUndefined(templateFileResolved)) {
        throw new Error('Template file not found: ' + templateFile);
      }

      if (_.isUndefined(dataFileResolved)) {
        throw new Error('Data file not found: ' + dataFile);
      }

      var str = fs.readFileSync(dataFileResolved, 'utf8');
      var data = yaml.safeLoad(str);

      if (dataFilter) {
        data = _.filter(data, dataFilter);
      }

      str = nunjucks.render(templateFileResolved, {
        data: data
      });

      return new nunjucks.runtime.SafeString(str);
    };
  };

  nunjucksEnv.addExtension('data', new DataExtension);
}

Toco.prototype = {
  constructor: Toco,
  /**
   * Initialises markdown parser.
   */
  initMdParser: function () {
    if (this.flags.initMdParser) {
      return;
    }

    var md = new Remarkable({
      html: true
    });

    md.config = this.config;

{'fraction_non_alphanumeric': 0.15702982349360925, 'fraction_numerical': 0.001978088861838101, 'mean_word_length': 3.3073394495412844, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5745607', 'n_tokens_mistral': 2182, 'n_tokens_neox': 2119, 'n_words': 485}