<commit_before>import * as DelimitedBlocks from './delimitedblocks'
import * as Macros from './macros'
import * as Options from './options'
import * as Spans from './spans'

export interface ExpansionOptions {
  [key: string]: boolean | undefined

  // Processing priority (highest to lowest): container, skip, spans and specials.
  // If spans is true then both spans and specials are processed.
  // They are assumed false if they are not explicitly defined.
  // If a custom filter is specified their use depends on the filter.
  macros?: boolean
  container?: boolean
  skip?: boolean
  spans?: boolean   // Span substitution also expands special characters.
  specials?: boolean
}

// http://stackoverflow.com/questions/3561493/is-there-a-regexp-escape-function-in-javascript
export function escapeRegExp(s: string): string {
  return s.replace(/[\-\/\\^$*+?.()|\[\]{}]/g, '\\$&')
}

export function replaceSpecialChars(s: string): string {
  return s.replace(/&/g, '&amp;')
    .replace(/>/g, '&gt;')
    .replace(/</g, '&lt;')
}

// Replace pattern '$1' or '$$1', '$2' or '$$2'... in `replacement` with corresponding match groups
// from `match`. If pattern starts with one '$' character add specials to `expansionOptions`,
// if it starts with two '$' characters add spans to `expansionOptions`.
export function replaceMatch(match: RegExpExecArray,
                             replacement: string,
                             expansionOptions: ExpansionOptions = {}): string {
  return replacement.replace(/(\${1,2})(\d)/g, function (): string {
    // Replace $1, $2 ... with corresponding match groups.
    if (arguments[1] === '$$') {
      expansionOptions.spans = true
    }
    else {
      expansionOptions.specials = true
    }
    let i = Number(arguments[2])  // match group number.
    let text = match[i]           // match group text.
    return replaceInline(text, expansionOptions)
  })
}

// Shallow object clone.
export function copy(source: any): any {
  let result: any = {}
  for (let key in source) {
    if (source.hasOwnProperty(key)) {
      result[key] = source[key]
    }
  }
  return result
}

// Copy properties in source object to target object.
export function merge(target: any, source: any): void {
  for (let key in source) {
    target[key] = source[key]
  }
}

// Replace the inline elements specified in options in text and return the result.
export function replaceInline(text: string, expansionOptions: ExpansionOptions): string {
  if (expansionOptions.macros) {
    text = Macros.render(text)
    text = text === null ? '' : text
  }
  // Spans also expand special characters.
  if (expansionOptions.spans) {
    text = Spans.render(text)
  }
  else if (expansionOptions.specials) {
    text = replaceSpecialChars(text)
  }
  return text
}

// Global Block Attributes state (namespace "singleton", see http://stackoverflow.com/a/30174360).
export namespace BlockAttributes {
  export let classes: string     // Space separated HTML class names.
  export let id: string          // HTML element id.
  export let css: string         // HTML CSS styles.
  export let attributes: string  // Other HTML element attributes.
  export let options: ExpansionOptions

  let ids: string[] // List of allocated HTML ids.

  export function init(): void {
    classes = ''
    id = ''
    css = ''
    attributes = ''
    options = {}
    ids = []
  }

  export function parse(match: RegExpExecArray): boolean {
    // Parse Block Attributes.
    // class names = $1, id = $2, css-properties = $3, html-attributes = $4, block-options = $5
    let text = match[0]
    text = replaceInline(text, {macros: true})
    let m = /^\\?\.((?:\s*[a-zA-Z][\w\-]*)+)*(?:\s*)?(#[a-zA-Z][\w\-]*\s*)?(?:\s*)?(?:"(.+?)")?(?:\s*)?(\[.+])?(?:\s*)?([+-][ \w+-]+)?$/.exec(text)
    if (!m) {
      return false
    }
    if (!Options.skipBlockAttributes()) {
      if (m[1]) { // HTML element class names.
        classes += ' ' + m[1].trim()
        classes = classes.trim()
      }
      if (m[2]) { // HTML element id.
        id = m[2].trim().slice(1)
      }
      if (m[3]) { // CSS properties.
        if (css && css.substr(-1) !== ';') css += ';'
        css += ' ' + m[3].trim()
        css = css.trim()
      }
      if (m[4] && !Options.isSafeModeNz()) { // HTML attributes.
        attributes += ' ' + m[4].slice(1, m[4].length - 1).trim()
        attributes = attributes.trim()
      }
      DelimitedBlocks.setBlockOptions(options, m[5])
    }
    return true
  }

  // Inject HTML attributes from attrs into the opening tag.
  // Consume HTML attributes unless the 'tag' argument is blank.
  export function inject(tag: string): string {
    if (!tag) {
      return tag
    }
    let attrs = ''
    if (classes) {
      if (/class=".*?"/i.test(tag)) {
        // Inject class names into existing class attribute.
        tag = tag.replace(/class="(.*?)"/i, `class="${classes} $1"`)
      }
      else {
        attrs = `class="${classes}"`
      }
    }
    if (id) {
      id = id.toLowerCase()
      if (ids.indexOf(id) > -1 || /id=".*?"/i.test(tag)) {
        Options.errorCallback(`duplicate 'id' attribute: ${id}`)
      }
      else {
        ids.push(id)
        attrs += ` id="${id}"`
      }
    }
    if (css) {
      if (/style=".*?"/i.test(tag)) {
        // Inject CSS styles into existing style attribute.
        tag = tag.replace(/style="(.*?)"/i, function (match: string, p1: string): string {
          p1 = p1.trim()
          if (p1 && p1.substr(-1) !== ';') p1 += ';'
          return `style="${p1} ${css}"`
        })
      }
      else {
        attrs += ` style="${css}"`
      }
    }
    if (attributes) {
      attrs += ' ' + attributes
    }
    attrs = attrs.trim()
    if (attrs) {
      let match = tag.match(/^<([a-zA-Z]+|h[1-6])(?=[ >])/)
      if (match) {
        let before = tag.slice(0, match[0].length)
        let after = tag.slice(match[0].length)
        tag = before + ' ' + attrs + after
      }
    }
    // Consume the attributes.
    classes = ''<commit_msg>feature: Duplicate HTML IDs are processed in output.<commit_after>import * as DelimitedBlocks from './delimitedblocks'
import * as Macros from './macros'
import * as Options from './options'
import * as Spans from './spans'

export interface ExpansionOptions {
  [key: string]: boolean | undefined

  // Processing priority (highest to lowest): container, skip, spans and specials.
  // If spans is true then both spans and specials are processed.
  // They are assumed false if they are not explicitly defined.
  // If a custom filter is specified their use depends on the filter.
  macros?: boolean
  container?: boolean
  skip?: boolean
  spans?: boolean   // Span substitution also expands special characters.
  specials?: boolean
}

// http://stackoverflow.com/questions/3561493/is-there-a-regexp-escape-function-in-javascript
export function escapeRegExp(s: string): string {
  return s.replace(/[\-\/\\^$*+?.()|\[\]{}]/g, '\\$&')
}

export function replaceSpecialChars(s: string): string {
  return s.replace(/&/g, '&amp;')
    .replace(/>/g, '&gt;')
    .replace(/</g, '&lt;')
}

// Replace pattern '$1' or '$$1', '$2' or '$$2'... in `replacement` with corresponding match groups
// from `match`. If pattern starts with one '$' character add specials to `expansionOptions`,
// if it starts with two '$' characters add spans to `expansionOptions`.
export function replaceMatch(match: RegExpExecArray,
                             replacement: string,
                             expansionOptions: ExpansionOptions = {}): string {
  return replacement.replace(/(\${1,2})(\d)/g, function (): string {
    // Replace $1, $2 ... with corresponding match groups.
    if (arguments[1] === '$$') {
      expansionOptions.spans = true
    }
    else {
      expansionOptions.specials = true
    }
    let i = Number(arguments[2])  // match group number.
    let text = match[i]           // match group text.
    return replaceInline(text, expansionOptions)
  })
}

// Shallow object clone.
export function copy(source: any): any {
  let result: any = {}
  for (let key in source) {
    if (source.hasOwnProperty(key)) {
      result[key] = source[key]
    }
  }
  return result
}

// Copy properties in source object to target object.
export function merge(target: any, source: any): void {
  for (let key in source) {
    target[key] = source[key]
  }
}

// Replace the inline elements specified in options in text and return the result.
export function replaceInline(text: string, expansionOptions: ExpansionOptions): string {
  if (expansionOptions.macros) {
    text = Macros.render(text)
    text = text === null ? '' : text
  }
  // Spans also expand special characters.
  if (expansionOptions.spans) {
    text = Spans.render(text)
  }
  else if (expansionOptions.specials) {
    text = replaceSpecialChars(text)
  }
  return text
}

// Global Block Attributes state (namespace "singleton", see http://stackoverflow.com/a/30174360).
export namespace BlockAttributes {
  export let classes: string     // Space separated HTML class names.
  export let id: string          // HTML element id.
  export let css: string         // HTML CSS styles.
  export let attributes: string  // Other HTML element attributes.
  export let options: ExpansionOptions

  let ids: string[] // List of allocated HTML ids.

  export function init(): void {
    classes = ''
    id = ''
    css = ''
    attributes = ''
    options = {}
    ids = []
  }

  export function parse(match: RegExpExecArray): boolean {
    // Parse Block Attributes.
    // class names = $1, id = $2, css-properties = $3, html-attributes = $4, block-options = $5
    let text = match[0]
    text = replaceInline(text, {macros: true})
    let m = /^\\?\.((?:\s*[a-zA-Z][\w\-]*)+)*(?:\s*)?(#[a-zA-Z][\w\-]*\s*)?(?:\s*)?(?:"(.+?)")?(?:\s*)?(\[.+])?(?:\s*)?([+-][ \w+-]+)?$/.exec(text)
    if (!m) {
      return false
    }
    if (!Options.skipBlockAttributes()) {
      if (m[1]) { // HTML element class names.
        classes += ' ' + m[1].trim()
        classes = classes.trim()
      }
      if (m[2]) { // HTML element id.
        id = m[2].trim().slice(1)
      }
      if (m[3]) { // CSS properties.
        if (css && css.substr(-1) !== ';') css += ';'
        css += ' ' + m[3].trim()
        css = css.trim()
      }
      if (m[4] && !Options.isSafeModeNz()) { // HTML attributes.
        attributes += ' ' + m[4].slice(1, m[4].length - 1).trim()
        attributes = attributes.trim()
      }
      DelimitedBlocks.setBlockOptions(options, m[5])
    }
    return true
  }

  // Inject HTML attributes from attrs into the opening tag.
  // Consume HTML attributes unless the 'tag' argument is blank.
  export function inject(tag: string): string {
    if (!tag) {
      return tag
    }
    let attrs = ''
    if (classes) {
      if (/class=".*?"/i.test(tag)) {
        // Inject class names into existing class attribute.
        tag = tag.replace(/class="(.*?)"/i, `class="${classes} $1"`)
      }
      else {
        attrs = `class="${classes}"`
      }
    }
    if (id) {
      id = id.toLowerCase()
      let has_id = /id=".*?"/i.test(tag)
      if (has_id || ids.indexOf(id) > -1) {
        Options.errorCallback(`duplicate 'id' attribute: ${id}`)
      }
      else {
        ids.push(id)
      }
      if (!has_id) {
        attrs += ` id="${id}"`
      }
    }
    if (css) {
      if (/style=".*?"/i.test(tag)) {
        // Inject CSS styles into existing style attribute.
        tag = tag.replace(/style="(.*?)"/i, function (match: string, p1: string): string {
          p1 = p1.trim()
          if (p1 && p1.substr(-1) !== ';') p1 += ';'
          return `style="${p1} ${css}"`
        })
      }
      else {
        attrs += ` style="${css}"`
      }
    }
    if (attributes) {
      attrs += ' ' + attributes
    }
    attrs = attrs.trim()
    if (attrs) {
      let match = tag.match(/^<([a-zA-Z]+|h[1-6])(?=[ >])/)
      if (match) {
        let before = tag.slice(0, match[0].length)
        let after = tag.slice(match[0].length)
        tag = before + ' ' + attrs + after
      }
    }
    // Consume the attributes.
    classes = ''
{'fraction_non_alphanumeric': 0.16828266228430566, 'fraction_numerical': 0.009860312243221035, 'mean_word_length': 2.394979079497908, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5065909', 'n_tokens_mistral': 3872, 'n_tokens_neox': 3826, 'n_words': 1088}