<commit_before>import asciimath2latex from 'asciimath-to-latex'
import katex from 'katex'
import hljs from 'highlight.js'

import Engine from './engine'

let options = {
  html: true,
  xhtmlOut: true, // <br /> instead of <br>
  linkify: true
}
let extensions = [
  'mark', 'ins', 'sub', 'sup', 'deflist', 'abbr', 'footnote', 'container', 'github-toc',
  'emoji', 'font-awesome', 'task-list', 'source-map'
]
let mdc = new Engine(options, extensions).mdc

// inline math
mdc.renderer.rules.code_inline = function (tokens, idx) {
  let code = tokens[idx].content
  if (code.startsWith('@') && code.endsWith('@')) {
    code = '$' + asciimath2latex(code.substr(1, code.length - 2)) + '$'
  }
  if (code.startsWith('$') && code.endsWith('$')) { // inline math
    code = code.substr(1, code.length - 2)
    try {
      return katex.renderToString(code)
    } catch (err) {
      return `<code>${err}</code>`
    }
  }
  return `<code>${mdc.utils.escapeHtml(code)}</code>` // not math
}

// math block
mdc.math_block = function (code) {
  let tex = ''
  code.split(/(?:\n\s*){2,}/).forEach(function (line) { // consecutive new lines means a new formula
    try {
      tex += katex.renderToString(line.trim(), { displayMode: true })
    } catch (err) {
      tex += `<pre>${err}</pre>`
    }
  })
  return `<div>${tex}</div>`
}

// chart block
mdc.chart_block = function (code) {
  try {
    let json = JSON.parse(code)
    return `<canvas class="chartjs">${JSON.stringify(json)}</canvas>`
  } catch (e) { // JSON.parse exception
    return `<pre>${e}</pre>`
  }
}

// placeholder for mermaid
mdc.mermaid_charts = function (code) {
  return `<div class="mermaid">${code}</div>`
}

// fence block
mdc.renderer.rules.fence = function (tokens, idx) {
  let token = tokens[idx]
  let code = token.content.trim()
  if (token.info === 'math' || token.info === 'katex') { // math
    return mdc.math_block(code)
  }
  if (/^ascii-?math/i.test(token.info)) {
    code = code.split(/(?:\n\s*){2,}/).map(function (item) { return asciimath2latex(item) }).join('\n\n')
    return mdc.math_block(code)
  }
  if (token.info === 'chart') { // chart
    return mdc.chart_block(code)
  }
  if (token.info === 'mermaid') { // mermaid
    return mdc.mermaid_charts(code) // mermaid
  }<commit_msg>Fix code style of index-node<commit_after>import asciimath2latex from 'asciimath-to-latex'
import katex from 'katex'
import hljs from 'highlight.js'

import Engine from './engine'

let options = {
  html: true,
  xhtmlOut: true, // <br /> instead of <br>
  linkify: true
}
let extensions = [
  'mark', 'ins', 'sub', 'sup', 'deflist', 'abbr', 'footnote', 'container', 'github-toc',
  'emoji', 'font-awesome', 'task-list', 'source-map'
]
let mdc = new Engine(options, extensions).mdc

// inline math
mdc.renderer.rules.code_inline = (tokens, idx, options, env, slf) => {
  let code = tokens[idx].content
  if (code.startsWith('@') && code.endsWith('@')) {
    code = '$' + asciimath2latex(code.substr(1, code.length - 2)) + '$'
  }
  if (code.startsWith('$') && code.endsWith('$')) { // inline math
    code = code.substr(1, code.length - 2)
    try {
      return katex.renderToString(code)
    } catch (err) {
      return `<code>${err}</code>`
    }
  }
  return `<code>${mdc.utils.escapeHtml(code)}</code>` // not math
}

// math block
mdc.math_block = (code) => {
  let tex = ''
  code.split(/(?:\n\s*){2,}/).forEach((line) => { // consecutive new lines means a new formula
    try {
      tex += katex.renderToString(line.trim(), { displayMode: true })
    } catch (err) {
      tex += `<pre>${err}</pre>`
    }
  })
  return `<div>${tex}</div>`
}

// chart block
mdc.chart_block = (code) => {
  try {
    let json = JSON.parse(code)
    return `<canvas class="chartjs">${JSON.stringify(json)}</canvas>`
  } catch (e) { // JSON.parse exception
    return `<pre>${e}</pre>`
  }
}

// placeholder for mermaid
mdc.mermaid_charts = (code) => {
  return `<div class="mermaid">${code}</div>`
}

// fence block
mdc.renderer.rules.fence = (tokens, idx, options, env, slf) => {
  let token = tokens[idx]
  let code = token.content.trim()
  if (token.info === 'math' || token.info === 'katex') { // math
    return mdc.math_block(code)
  }
  if (/^ascii-?math/i.test(token.info)) {
    code = code.split(/(?:\n\s*){2,}/).map((item) => { return asciimath2latex(item) }).join('\n\n')
    return mdc.math_block(code)
  }
  if (token.info === 'chart') { // chart
    return mdc.chart_block(code)
  }
  if (token.info === 'mermaid') { // mermaid
    return mdc.mermaid_charts(code) // mermaid
  }
{'fraction_non_alphanumeric': 0.20508883527089275, 'fraction_numerical': 0.0039482342618995395, 'mean_word_length': 3.2857142857142856, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5585908', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1715, 'n_words': 365}