<commit_before>/*
 This module renders text containing Quote and Replacement elements.

 Quote and replacement processing involves splitting the source text into
 fragments where a quote or a replacement occurs then splicing the fragments
 containing HTML markup into the breaks.  A fragment is flagged as 'done' to
 exclude it from further substitutions.

 Once all quotes and replacements are processed fragments not yet flagged as
 'done' have special characters (&, <, >) replaced with corresponding special
 character entities. The fragments are then reassembled (defraged) into a
 resultant HTML string.
 */

import * as utils from './utils'
import * as quotes from './quotes'
import * as replacements from './replacements'

interface Fragment {
  text: string
  done: boolean
  verbatim?: string   // Replacements text rendered verbatim.
}

export function render(source: string): string {
  let result: string
  result = preReplacements(source)
  let fragments: Fragment[] = [{text: result, done: false}]
  fragQuotes(fragments)
  fragSpecials(fragments)
  result = defrag(fragments)
  return postReplacements(result)
}

/*
 Replace render() with this function to process replacements *after* quotes (pre version 5 behaviour).
 */
//export function render(source: string): string {
//  let fragments: Fragment[] = [{text: source, done: false}]
//  fragQuotes(fragments)
//  fragReplacements(fragments)
//  fragSpecials(fragments)
//  return defrag(fragments)
//}

// Converts fragments to a string.
function defrag(fragments: Fragment[]): string {
  return fragments.reduce((result, fragment) => result + fragment.text, '')
}

function fragQuotes(fragments: Fragment[]): void {
  let matchRe = quotes.quotesRe
  let fragmentIndex = 0
  let fragment = fragments[fragmentIndex]
  let fragmentIsDone: boolean
  let match: RegExpExecArray
  matchRe.lastIndex = 0
  while (true) {
    if (fragment.done) {
      fragmentIsDone = true
    }
    else {
      match = matchRe.exec(fragment.text)
      fragmentIsDone = !match
    }
    if (fragmentIsDone) {
      fragmentIndex++
      if (fragmentIndex >= fragments.length) {
        break // All fragments processed.
      }
      fragment = fragments[fragmentIndex]
      matchRe.lastIndex = 0
      continue
    }
    if (match[0][0] === '\\') {
      // Restart search after opening quote.
      matchRe.lastIndex = match.index + match[1].length + 1
      continue
    }
    // Arrive here if we have a matched quote.
    let def = quotes.getDefinition(match[1])
/*
 Unused
    if (def.verify && !def.verify(match, matchRe)) {
      // Restart search after opening quote.
      matchRe.lastIndex = match.index + match[1].length + 1
      continue
    }
*/
    // Check for same closing quote one character further to the right.
    while (fragment.text[matchRe.lastIndex] === match[1][0]) {
      // Move to closing quote one character to right.
      match[2] += match[1][0]
      matchRe.lastIndex += 1
    }
    // The quotes splits the fragment into 5 fragments.
    let before = match.input.slice(0, match.index)
    let quoted = match[2]
    let after = match.input.slice(matchRe.lastIndex)
    fragments.splice(fragmentIndex, 1,
      {text: before, done: false},
      {text: def.openTag, done: true},
      {text: quoted, done: false},
      {text: def.closeTag, done: true},
      {text: after, done: false}
    )
    // Move to 'quoted' fragment.
    fragmentIndex += 2
    fragment = fragments[fragmentIndex]
    if (!def.spans) {
      fragment.text = utils.replaceSpecialChars(fragment.text)
      fragment.text = fragment.text.replace('\u0000', '\u0001')   // Use verbatim replacement.
      fragment.done = true
      // Move to 'after' fragment.
      fragmentIndex += 2
      fragment = fragments[fragmentIndex]
    }
    matchRe.lastIndex = 0
  }
  // Strip backlash from escaped quotes in non-done fragments.
  fragments
    .filter(fragment => !fragment.done)
    .forEach(fragment => fragment.text = quotes.unescape(fragment.text))
}

// Replacements fragments set by `preReplacements()`, used by `postReplacements()`.
let savedReplacements: Fragment[]

// Return text with replacements replaced with placeholders (see `postReplacements()`).
function preReplacements(text: string): string {
  savedReplacements = []
  let fragments: Fragment[] = [{text: text, done: false}]<commit_msg>fix: Replace more than one verbatim replacement inside code quotes.<commit_after>/*
 This module renders text containing Quote and Replacement elements.

 Quote and replacement processing involves splitting the source text into
 fragments where a quote or a replacement occurs then splicing the fragments
 containing HTML markup into the breaks.  A fragment is flagged as 'done' to
 exclude it from further substitutions.

 Once all quotes and replacements are processed fragments not yet flagged as
 'done' have special characters (&, <, >) replaced with corresponding special
 character entities. The fragments are then reassembled (defraged) into a
 resultant HTML string.
 */

import * as utils from './utils'
import * as quotes from './quotes'
import * as replacements from './replacements'

interface Fragment {
  text: string
  done: boolean
  verbatim?: string   // Replacements text rendered verbatim.
}

export function render(source: string): string {
  let result: string
  result = preReplacements(source)
  let fragments: Fragment[] = [{text: result, done: false}]
  fragQuotes(fragments)
  fragSpecials(fragments)
  result = defrag(fragments)
  return postReplacements(result)
}

/*
 Replace render() with this function to process replacements *after* quotes (pre version 5 behaviour).
 */
//export function render(source: string): string {
//  let fragments: Fragment[] = [{text: source, done: false}]
//  fragQuotes(fragments)
//  fragReplacements(fragments)
//  fragSpecials(fragments)
//  return defrag(fragments)
//}

// Converts fragments to a string.
function defrag(fragments: Fragment[]): string {
  return fragments.reduce((result, fragment) => result + fragment.text, '')
}

function fragQuotes(fragments: Fragment[]): void {
  let matchRe = quotes.quotesRe
  let fragmentIndex = 0
  let fragment = fragments[fragmentIndex]
  let fragmentIsDone: boolean
  let match: RegExpExecArray
  matchRe.lastIndex = 0
  while (true) {
    if (fragment.done) {
      fragmentIsDone = true
    }
    else {
      match = matchRe.exec(fragment.text)
      fragmentIsDone = !match
    }
    if (fragmentIsDone) {
      fragmentIndex++
      if (fragmentIndex >= fragments.length) {
        break // All fragments processed.
      }
      fragment = fragments[fragmentIndex]
      matchRe.lastIndex = 0
      continue
    }
    if (match[0][0] === '\\') {
      // Restart search after opening quote.
      matchRe.lastIndex = match.index + match[1].length + 1
      continue
    }
    // Arrive here if we have a matched quote.
    let def = quotes.getDefinition(match[1])
/*
 Unused
    if (def.verify && !def.verify(match, matchRe)) {
      // Restart search after opening quote.
      matchRe.lastIndex = match.index + match[1].length + 1
      continue
    }
*/
    // Check for same closing quote one character further to the right.
    while (fragment.text[matchRe.lastIndex] === match[1][0]) {
      // Move to closing quote one character to right.
      match[2] += match[1][0]
      matchRe.lastIndex += 1
    }
    // The quotes splits the fragment into 5 fragments.
    let before = match.input.slice(0, match.index)
    let quoted = match[2]
    let after = match.input.slice(matchRe.lastIndex)
    fragments.splice(fragmentIndex, 1,
      {text: before, done: false},
      {text: def.openTag, done: true},
      {text: quoted, done: false},
      {text: def.closeTag, done: true},
      {text: after, done: false}
    )
    // Move to 'quoted' fragment.
    fragmentIndex += 2
    fragment = fragments[fragmentIndex]
    if (!def.spans) {
      fragment.text = utils.replaceSpecialChars(fragment.text)
      fragment.text = fragment.text.replace(/\u0000/g, '\u0001')   // Use verbatim replacements.
      fragment.done = true
      // Move to 'after' fragment.
      fragmentIndex += 2
      fragment = fragments[fragmentIndex]
    }
    matchRe.lastIndex = 0
  }
  // Strip backlash from escaped quotes in non-done fragments.
  fragments
    .filter(fragment => !fragment.done)
    .forEach(fragment => fragment.text = quotes.unescape(fragment.text))
}

// Replacements fragments set by `preReplacements()`, used by `postReplacements()`.
let savedReplacements: Fragment[]

// Return text with replacements replaced with placeholders (see `postReplacements()`).
function preReplacements(text: string): string {
  savedReplacements = []
  let fragments: Fragment[] = [{text: text, done: false}]
{'fraction_non_alphanumeric': 0.10888382687927108, 'fraction_numerical': 0.007289293849658315, 'mean_word_length': 3.293887530562347, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4957864', 'n_tokens_mistral': 2557, 'n_tokens_neox': 2457, 'n_words': 818}