<commit_before>const path = require('path')
const _ = require('lodash')
const nunjucks = require('nunjucks')
const Promise = require('bluebird')
const marked = require('marked')
const { detergent } = require('detergent')
const fse = require('fs-extra')

const utils = require('./utils')
const logger = require('./logger')

// configure Nunjucks
const paths = [utils.getTemplatesPath(), utils.getComponentsPath()].filter(
  Boolean
)
nunjucks.configure(paths, { autoescape: false })

// create a renderer for marked
// do not wrap paragraphs in <p>
// do not automatically convert http://... to <a>s
const renderer = new marked.Renderer()
const defaultRenderer = new marked.Renderer()
renderer.paragraph = text => text
renderer.link = (href, title, text) => {
  if (href === text) {
    return href
  }
  return defaultRenderer.link(href, title, text)
}

class NunjucksCompiler {
  constructor(tplPath) {
    this.tplPath = tplPath
  }

  async get() {
    try {
      const context = await this.requireContext()

      // render HTML and text versions with Nunjucks
      // HTML version context is parsed as Markdown
      const [html, text] = await Promise.all([
        this.render(
          path.join(this.tplPath, 'html.njk'),
          this.cleanContext(context, true)
        ),
        this.render(
          path.join(this.tplPath, 'text.njk'),
          this.cleanContext(context)
        )
      ])
      return { html, text }
    } catch (err) {
      logger.error(err)
    }
  }

  async requireContext() {
    try {
      const contextStr = fse.readFileSync(
        path.join(this.tplPath, 'context.json'),
        'utf-8'
      )
      return contextStr ? JSON.parse(contextStr) : {}
    } catch (e) {
      // ignore missing context file
      return {}
    }
  }

  render(tplPath, context) {
    return new Promise((resolve, reject) => {
      nunjucks.render(tplPath, context, (err, res) => {
        if (err) reject(err)
        else resolve(res)
      })
    })<commit_msg>fix: rebuild Nunjucks templates every time<commit_after>const path = require('path')
const _ = require('lodash')
const nunjucks = require('nunjucks')
const Promise = require('bluebird')
const marked = require('marked')
const { detergent } = require('detergent')
const fse = require('fs-extra')

const utils = require('./utils')
const logger = require('./logger')

// create a renderer for marked
// do not wrap paragraphs in <p>
// do not automatically convert http://... to <a>s
const renderer = new marked.Renderer()
const defaultRenderer = new marked.Renderer()
renderer.paragraph = text => text
renderer.link = (href, title, text) => {
  if (href === text) {
    return href
  }
  return defaultRenderer.link(href, title, text)
}

class NunjucksCompiler {
  constructor(tplPath) {
    this.tplPath = tplPath
  }

  async get() {
    try {
      const context = await this.requireContext()

      // render HTML and text versions with Nunjucks
      // HTML version context is parsed as Markdown
      const [html, text] = await Promise.all([
        this.render(
          path.join(this.tplPath, 'html.njk'),
          this.cleanContext(context, true)
        ),
        this.render(
          path.join(this.tplPath, 'text.njk'),
          this.cleanContext(context)
        )
      ])
      return { html, text }
    } catch (err) {
      logger.error(err)
    }
  }

  async requireContext() {
    try {
      const contextStr = fse.readFileSync(
        path.join(this.tplPath, 'context.json'),
        'utf-8'
      )
      return contextStr ? JSON.parse(contextStr) : {}
    } catch (e) {
      // ignore missing context file
      return {}
    }
  }

  configureNunjucks() {
    const paths = [utils.getTemplatesPath(), utils.getComponentsPath()].filter(
      Boolean
    )
    nunjucks.configure(paths, { autoescape: false })
  }

  render(tplPath, context) {
    return new Promise((resolve, reject) => {
      nunjucks.render(tplPath, context, (err, res) => {
        if (err) reject(err)
        else resolve(res)
      })
    })
{'fraction_non_alphanumeric': 0.1328221617610685, 'fraction_numerical': 0.0004946821667078902, 'mean_word_length': 2.6696914700544463, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1823348', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1240, 'n_words': 320}