<commit_before>var _ = require('lodash');
var cheerio = require('cheerio');
var fs = require('fs-extra');
var multiGlob = require('./util').multiGlob;
var nunjucks = require('nunjucks');
var path = require('path');
var Penrose = require('penrose').Penrose;
var Promise = require('bluebird');
var Remarkable = require('remarkable');
var runWebpack = require('./util').runWebpack;
var sanitizeFilename = require('sanitize-filename');
var toc = require('markdown-toc');
var vm = require('vm');
var webpack = require('webpack');
var yaml = require('js-yaml');
var yamlFrontMatter = require('yaml-front-matter');

// Builds.
var DEV = 'dev';
var PROD = 'prod';

// Event hooks.
var PRE_RENDER_HTML = 'preRenderHtml';
var POST_RENDER_HTML = 'postRenderHtml';
var POST_RENDER_TEMPLATE = 'postRenderTemplate';

Promise.promisifyAll(fs);

fs.existsAsync = Promise.promisify(function exists2(path, exists2callback) {
  fs.exists(path, function callbackWrapper(exists) {
    exists2callback(null, exists);
  });
});

var contentDirname = 'content';
var contentExtnames = [
  'md'
];

var dataDirname = 'data';
var dataExtnames = [
  'js',
  'json',
  'yaml',
  'yml'
];

var themesDirname = 'themes';
var templatesDirname = 'templates';
var partialsDirname = '_partials';
var templateExtnames = [
  'njk'
];

var renderExtname = 'html';

var extnameFormatMap = {
  'js': 'js',
  'json': 'json',
  'yaml': 'yaml',
  'yml': 'yaml'
};

/**
 *
 * @param  {String} file File basename, file basename with extension, or path
 * @param  {Array} dirs
 * @param  {Array} extnames
 * @return {String}
 */
function findFile(file, dirs, extnames) {
  var i, il, j, jl, dir, extname, str;

  if (file.substring(0, 1) === '/' || file.substring(0, 2) === './' || file.substring(0, 3) === '../') {
    if (fs.existsSync(file)) {
      return file;
    }

    for (i = 0, il = extnames.length; i < il; i++) {
      extname = extnames[i];
      str = file + extname;
      if (fs.existsSync(str)) {
        return str;
      }
    }

    return;
  }

  for (i = 0, il = dirs.length; i < il; i++) {
    dir = dirs[i];
    str = path.resolve(dir, file);
    if (fs.existsSync(str)) {
      return str;
    }
  }

  for (i = 0, il = dirs.length; i < il; i++) {
    dir = dirs[i];
    for (j = 0, jl = extnames.length; j < jl; j++) {
      extname = extnames[j];
      str = path.resolve(dir, file + extname);
      if (fs.existsSync(str)) {
        return str;
      }
    }
  }
}

/**
 *
 * @param  {String} str
 * @return {Array}
 */
function getModifiers(str) {
  return _.difference(_.trim(str, '-').split('--').slice(1), ['']);
}

/**
 *
 * @param  {String} str
 * @return {String}
 */
function stripModifiers(str) {
  return _.trim(str, '-').split('--')[0];
}

/**
 *
 * @param  {Object} config
 * @return {Lichen}
 */
function Lichen(config) {
  this.config = _.assign({}, {
    data: {},
    env: {
      dev: false // {Boolean} Set to `true` for production build
    },
    hooks: [],
    nunjucks: {
      filters: {}
    },
    patterns: {
      contentDistPath: '{{ file.dirname }}/{{ file.basename }}.{{ renderExtname }}'
    },
    penrose: undefined,
    remarkable: {
      plugins: {}
    },
    // @todo Use more appropriate/descriptive name than `themes`
    themes: [],
    // @see https://github.com/jonschlinkert/markdown-toc#options
    toc: {
      firsth1: true, // {Boolean} Exclude the first h1-level heading
      maxdepth: 6 // {Number} Use headings whose depth is at most maxdepth
    },
    webpack: undefined
  }, config);

  var findPartialTemplate = function (templateName) {
    return this.findPartialTemplate(templateName);
  }.bind(this);

  var templateDirs = [];

  if (_.isUndefined(this.config.templateDirs)) {
    templateDirs = templateDirs.concat([
      path.resolve(this.config.pages.src.path, themesDirname),
      path.resolve(this.config.pages.src.path, templatesDirname)
    ]);
  }
  else {
    templateDirs = templateDirs.concat(this.config.templateDirs);
  }

  templateDirs.push(path.resolve(__dirname, '..', templatesDirname));

  var nunjucksEnv = nunjucks.configure(templateDirs);

  _.forEach(this.config.nunjucks.filters, function (filter, filterName) {
    nunjucksEnv.addFilter(filterName, filter);
  });

  nunjucksEnv.addFilter('markdown', function (data) {
    var md = new Remarkable({
      html: true
    });
    return md.render(data);
  });

  nunjucksEnv.addFilter('remove', function (data, selector) {
    var $ = cheerio.load(data);
    $(selector).remove();
    return $.html();
  });

  nunjucksEnv.addFilter('sanitizeFilename', function (data) {
    return sanitizeFilename(data);
  });

  // Render partial template with data. Similar to `include` tag.
  var RenderExtension = function () {
    this.tags = ['render'];<commit_msg>Add kebabCase filter<commit_after>var _ = require('lodash');
var cheerio = require('cheerio');
var fs = require('fs-extra');
var multiGlob = require('./util').multiGlob;
var nunjucks = require('nunjucks');
var path = require('path');
var Penrose = require('penrose').Penrose;
var Promise = require('bluebird');
var Remarkable = require('remarkable');
var runWebpack = require('./util').runWebpack;
var sanitizeFilename = require('sanitize-filename');
var toc = require('markdown-toc');
var vm = require('vm');
var webpack = require('webpack');
var yaml = require('js-yaml');
var yamlFrontMatter = require('yaml-front-matter');

// Builds.
var DEV = 'dev';
var PROD = 'prod';

// Event hooks.
var PRE_RENDER_HTML = 'preRenderHtml';
var POST_RENDER_HTML = 'postRenderHtml';
var POST_RENDER_TEMPLATE = 'postRenderTemplate';

Promise.promisifyAll(fs);

fs.existsAsync = Promise.promisify(function exists2(path, exists2callback) {
  fs.exists(path, function callbackWrapper(exists) {
    exists2callback(null, exists);
  });
});

var contentDirname = 'content';
var contentExtnames = [
  'md'
];

var dataDirname = 'data';
var dataExtnames = [
  'js',
  'json',
  'yaml',
  'yml'
];

var themesDirname = 'themes';
var templatesDirname = 'templates';
var partialsDirname = '_partials';
var templateExtnames = [
  'njk'
];

var renderExtname = 'html';

var extnameFormatMap = {
  'js': 'js',
  'json': 'json',
  'yaml': 'yaml',
  'yml': 'yaml'
};

/**
 *
 * @param  {String} file File basename, file basename with extension, or path
 * @param  {Array} dirs
 * @param  {Array} extnames
 * @return {String}
 */
function findFile(file, dirs, extnames) {
  var i, il, j, jl, dir, extname, str;

  if (file.substring(0, 1) === '/' || file.substring(0, 2) === './' || file.substring(0, 3) === '../') {
    if (fs.existsSync(file)) {
      return file;
    }

    for (i = 0, il = extnames.length; i < il; i++) {
      extname = extnames[i];
      str = file + extname;
      if (fs.existsSync(str)) {
        return str;
      }
    }

    return;
  }

  for (i = 0, il = dirs.length; i < il; i++) {
    dir = dirs[i];
    str = path.resolve(dir, file);
    if (fs.existsSync(str)) {
      return str;
    }
  }

  for (i = 0, il = dirs.length; i < il; i++) {
    dir = dirs[i];
    for (j = 0, jl = extnames.length; j < jl; j++) {
      extname = extnames[j];
      str = path.resolve(dir, file + extname);
      if (fs.existsSync(str)) {
        return str;
      }
    }
  }
}

/**
 *
 * @param  {String} str
 * @return {Array}
 */
function getModifiers(str) {
  return _.difference(_.trim(str, '-').split('--').slice(1), ['']);
}

/**
 *
 * @param  {String} str
 * @return {String}
 */
function stripModifiers(str) {
  return _.trim(str, '-').split('--')[0];
}

/**
 *
 * @param  {Object} config
 * @return {Lichen}
 */
function Lichen(config) {
  this.config = _.assign({}, {
    data: {},
    env: {
      dev: false // {Boolean} Set to `true` for production build
    },
    hooks: [],
    nunjucks: {
      filters: {}
    },
    patterns: {
      contentDistPath: '{{ file.dirname }}/{{ file.basename }}.{{ renderExtname }}'
    },
    penrose: undefined,
    remarkable: {
      plugins: {}
    },
    // @todo Use more appropriate/descriptive name than `themes`
    themes: [],
    // @see https://github.com/jonschlinkert/markdown-toc#options
    toc: {
      firsth1: true, // {Boolean} Exclude the first h1-level heading
      maxdepth: 6 // {Number} Use headings whose depth is at most maxdepth
    },
    webpack: undefined
  }, config);

  var findPartialTemplate = function (templateName) {
    return this.findPartialTemplate(templateName);
  }.bind(this);

  var templateDirs = [];

  if (_.isUndefined(this.config.templateDirs)) {
    templateDirs = templateDirs.concat([
      path.resolve(this.config.pages.src.path, themesDirname),
      path.resolve(this.config.pages.src.path, templatesDirname)
    ]);
  }
  else {
    templateDirs = templateDirs.concat(this.config.templateDirs);
  }

  templateDirs.push(path.resolve(__dirname, '..', templatesDirname));

  var nunjucksEnv = nunjucks.configure(templateDirs);

  _.forEach(this.config.nunjucks.filters, function (filter, filterName) {
    nunjucksEnv.addFilter(filterName, filter);
  });

  nunjucksEnv.addFilter('kebabCase', function (data) {
    return _.kebabCase(data);
  });

  nunjucksEnv.addFilter('markdown', function (data) {
    var md = new Remarkable({
      html: true
    });
    return md.render(data);
  });

  nunjucksEnv.addFilter('remove', function (data, selector) {
    var $ = cheerio.load(data);
    $(selector).remove();
    return $.html();
  });

  nunjucksEnv.addFilter('sanitizeFilename', function (data) {
    return sanitizeFilename(data);
  });

  // Render partial template with data. Similar to `include` tag.
  var RenderExtension = function () {
    this.tags = ['render'];
{'fraction_non_alphanumeric': 0.17176324245879548, 'fraction_numerical': 0.0037317300715248264, 'mean_word_length': 3.267138434321097, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3655321', 'n_tokens_mistral': 3369, 'n_tokens_neox': 3295, 'n_words': 756}