<commit_before>const _ = require('lodash')
const md = require('markdown-it')()
const markdown = require('nunjucks-markdown')
const crumble = require('./crumble')
const render = require('./nunjucks-render')
const format = require('./format')
const urlify = require('./urlify')
const imageSize = require('./image-size')
const numbro = require('numbro')
const moment = require('moment')
const smartPlurals = require('smart-plurals')
const { join } = require('path')
const URI = require('urijs')
const urljoin = require('./urljoin')
const { cacheFunc } = require('./cache')
const { escape } = require('nunjucks/src/lib')
const { file: { expand }, log } = require('grunt')

module.exports = function (env) {
  // ==============================================================================
  // Extensions
  // ==============================================================================

  /**
   * Nunjucks extension for Markdown support
   * @example {% markdown %}Markdown _text_ goes **here**{% endmarkdown %}
  */
  markdown.register(env, md.render.bind(md))

  // ==============================================================================
  // Globals
  // ==============================================================================

  /**
   * Pass lodash inside Nunjucks
  */
  env.addGlobal('_', _)

  /**
   * Log specified to Grunt's console for debug purposes
   * @param {*} input Anything we want to log to console
   * @return {void} Logs to Grunt console
  */
  env.addGlobal('log', (...input) => console.log(...input))

  /**
   * Log specified to Grunt's console as warning message
   * @param {*} input Anything we want to log to console
   * @return {void} Logs to Grunt console
  */
  env.addGlobal('warn', function (...input) {
    log.error(...input, `[${this.ctx.PAGE.props.url}]`)
  })

  /**
   * Get list of files or directories inside specified directory
   * @param {string}          [path = '']                           Path where to look
   * @param {string|string[]} [pattern = '** /*']                   What should be matched
   * @param {string}          [filter = 'isFile']                   Type of entity which should be matched
   * @param {string}          [cwd = this.ctx.PATH.build.templates] Root for lookup
   * @return {array} Array of found files or directories
  */
  env.addGlobal('expand', function (path = '', pattern = '**/*', filter = 'isFile', cwd = this.ctx.PATH.build.templates) {
    return expand({ cwd: join(cwd, path), filter }, pattern)
  })

  /**
   * Define config in context of current template. If config has been already
   * defined on parent, it will extend it, unless `merge` set to false.
   * If no `value` will be provided, it will return value of specified property.
   * Works similar to `grunt.config()`
   * @param  {string|string[]} prop           Prop name or path on which should be set `value`
   * @param  {*}               [value]        Value to be set on specified `prop`
   * @param  {bool}            [merge = true] Should config extend already existing same prop or no
   * @return {*} Value of `prop` if no `value` specified
  */
  env.addGlobal('config', function (prop, value, merge = true) {
    const ctxValue = _.get(this.ctx, prop)

    // Get current contenxt value if no `value` provided
    if (value === undefined) return ctxValue

    if (!merge || !ctxValue) {
      _.set(this.ctx, prop, value)
      return
    }

    // If this isn't Object, nothing we can do here, exit without changes to context
    if (typeof value !== 'object') return

    // Shallow cloning prevents leaking when merging
    value = (_.isArray(value) && _.union([], value, ctxValue)) || _.merge({}, value, ctxValue)

    _.set(this.ctx, prop, value)
  })

  /**
   * Get properties of page and its childs from site Matter data
   * @param {string|string[]} path                        Path to page inside site Matter data.
   *                                                      Array of crumbs, dot-notation string or url.
   * @param {boolean}         [forceRender = true]        Render output with Nunjucks
   * @param {boolean}         [cached = true]             Use cached rendered version if available
   * @param {object}          [ctx = this.getVariables()] Nunjucks context for forced rendering
   * @return {object} Properties of the page, including its sub pages
   * @example
   *   getPage('blog.post')
   *   getPage('blog/post')
   *   getPage(['blog', 'post'])
  */
  env.addGlobal('getPage', function (path, forceRender = true, cached = true, ctx = this.getVariables()) {
    path = (path.includes('/') && crumble(path)) || path

    const renderData = (tmpl) => render(env, ctx, tmpl)

    const { SITE: { matter }, PAGE } = this.ctx
    const locale = PAGE.locale || PAGE.props.locale
    const data = (forceRender && cached && cacheFunc(renderData, `renderedMatter_${locale}`)(matter)) || matter
    let page = _.get(data, path)

    if (!page) {
      log.error(`[getPage] can not find \`${path}\` inside site Matter data [${this.ctx.PAGE.props.url}]`)
      return
    }

    if (forceRender) {
      page = cached ? page : renderData(page)
    }
<commit_msg>[modules] make locale in `getPage` optional<commit_after>const _ = require('lodash')
const md = require('markdown-it')()
const markdown = require('nunjucks-markdown')
const crumble = require('./crumble')
const render = require('./nunjucks-render')
const format = require('./format')
const urlify = require('./urlify')
const imageSize = require('./image-size')
const numbro = require('numbro')
const moment = require('moment')
const smartPlurals = require('smart-plurals')
const { join } = require('path')
const URI = require('urijs')
const urljoin = require('./urljoin')
const { cacheFunc } = require('./cache')
const { escape } = require('nunjucks/src/lib')
const { file: { expand }, log } = require('grunt')

module.exports = function (env) {
  // ==============================================================================
  // Extensions
  // ==============================================================================

  /**
   * Nunjucks extension for Markdown support
   * @example {% markdown %}Markdown _text_ goes **here**{% endmarkdown %}
  */
  markdown.register(env, md.render.bind(md))

  // ==============================================================================
  // Globals
  // ==============================================================================

  /**
   * Pass lodash inside Nunjucks
  */
  env.addGlobal('_', _)

  /**
   * Log specified to Grunt's console for debug purposes
   * @param {*} input Anything we want to log to console
   * @return {void} Logs to Grunt console
  */
  env.addGlobal('log', (...input) => console.log(...input))

  /**
   * Log specified to Grunt's console as warning message
   * @param {*} input Anything we want to log to console
   * @return {void} Logs to Grunt console
  */
  env.addGlobal('warn', function (...input) {
    log.error(...input, `[${this.ctx.PAGE.props.url}]`)
  })

  /**
   * Get list of files or directories inside specified directory
   * @param {string}          [path = '']                           Path where to look
   * @param {string|string[]} [pattern = '** /*']                   What should be matched
   * @param {string}          [filter = 'isFile']                   Type of entity which should be matched
   * @param {string}          [cwd = this.ctx.PATH.build.templates] Root for lookup
   * @return {array} Array of found files or directories
  */
  env.addGlobal('expand', function (path = '', pattern = '**/*', filter = 'isFile', cwd = this.ctx.PATH.build.templates) {
    return expand({ cwd: join(cwd, path), filter }, pattern)
  })

  /**
   * Define config in context of current template. If config has been already
   * defined on parent, it will extend it, unless `merge` set to false.
   * If no `value` will be provided, it will return value of specified property.
   * Works similar to `grunt.config()`
   * @param  {string|string[]} prop           Prop name or path on which should be set `value`
   * @param  {*}               [value]        Value to be set on specified `prop`
   * @param  {bool}            [merge = true] Should config extend already existing same prop or no
   * @return {*} Value of `prop` if no `value` specified
  */
  env.addGlobal('config', function (prop, value, merge = true) {
    const ctxValue = _.get(this.ctx, prop)

    // Get current contenxt value if no `value` provided
    if (value === undefined) return ctxValue

    if (!merge || !ctxValue) {
      _.set(this.ctx, prop, value)
      return
    }

    // If this isn't Object, nothing we can do here, exit without changes to context
    if (typeof value !== 'object') return

    // Shallow cloning prevents leaking when merging
    value = (_.isArray(value) && _.union([], value, ctxValue)) || _.merge({}, value, ctxValue)

    _.set(this.ctx, prop, value)
  })

  /**
   * Get properties of page and its childs from site Matter data
   * @param {string|string[]} path                        Path to page inside site Matter data.
   *                                                      Array of crumbs, dot-notation string or url.
   * @param {boolean}         [forceRender = true]        Render output with Nunjucks
   * @param {boolean}         [cached = true]             Use cached rendered version if available
   * @param {object}          [ctx = this.getVariables()] Nunjucks context for forced rendering
   * @return {object} Properties of the page, including its sub pages
   * @example
   *   getPage('blog.post')
   *   getPage('blog/post')
   *   getPage(['blog', 'post'])
  */
  env.addGlobal('getPage', function (path, forceRender = true, cached = true, ctx = this.getVariables()) {
    path = (path.includes('/') && crumble(path)) || path

    const renderData = (tmpl) => render(env, ctx, tmpl)

    const { SITE: { matter }, PAGE } = this.ctx
    const locale = PAGE && (PAGE.locale || PAGE.props.locale)
    const data = (forceRender && cached && cacheFunc(renderData, `renderedMatter_${locale}`)(matter)) || matter
    let page = _.get(data, path)

    if (!page) {
      log.error(`[getPage] can not find \`${path}\` inside site Matter data [${this.ctx.PAGE.props.url}]`)
      return
    }

    if (forceRender) {
      page = cached ? page : renderData(page)
    }

{'fraction_non_alphanumeric': 0.19371626831148805, 'fraction_numerical': 0.0, 'mean_word_length': 2.930681818181818, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1759114', 'n_tokens_mistral': 2959, 'n_tokens_neox': 2843, 'n_words': 950}