<commit_before>/// <reference path="references.ts" />

module Rimu.DelimitedBlocks {

  // Multi-line block element definition.
  export interface Definition {
    name?: string;       // Optional unique identifier.
    openMatch: RegExp;   // $1 (if defined) is prepended to block content.
    closeMatch?: RegExp; // $1 (if defined) is appended to block content. If closeMatch is undefined then it must match opening delimiter.
    openTag: string;
    closeTag: string;
    filter?: (text: string, match?: string[], expansionOptions?: ExpansionOptions) => string;
    verify?: (match: string[]) => boolean;  // Additional match verification checks.
    expansionOptions: ExpansionOptions;
  }

  var defs: Definition[] = [
    // Delimited blocks cannot be escaped with a backslash.

    // Macro definition block.
    {
      openMatch: Macros.MACRO_DEF_OPEN,    // $1 is first line of macro.
      closeMatch: Macros.MACRO_DEF_CLOSE,  // $1 is last line of macro.
      openTag: '',
      closeTag: '',
      expansionOptions: {
        macros: true
      },
      filter: function (text: string, match: string[], expansionOptions: ExpansionOptions): string {
        // Set macro.
        // Get the macro name from the match in the first line of the block.
        var name = match[0].match(/^\{([\w\-]+)\}/)[1];
        text = text.replace(/' *\\\n/g, '\'\n');        // Unescape line-continuations.
        text = text.replace(/(' *[\\]+)\\\n/g, '$1\n'); // Unescape escaped line-continuations.
        text = replaceInline(text, expansionOptions);   // Expand macro invocations.
        Macros.setValue(name, text);
        return '';
      }
    },
    // Comment block.
    {
      name: 'comment',
      openMatch: /^\\?\/\*+$/,
      closeMatch: /^\*+\/$/,
      openTag: '',
      closeTag: '',
      expansionOptions: {
        skip: true,
        specials: true // Fall-back if skip is disabled.
      }
    },
    // Division block.
    {
      name: 'division',
      openMatch: /^\\?\.{2,}$/,
      openTag: '<div>',
      closeTag: '</div>',
      expansionOptions: {
        container: true,
        specials: true // Fall-back if container is disabled.
      }
    },
    // Quote block.
    {
      name: 'quote',
      openMatch: /^\\?"{2,}$/,
      openTag: '<blockquote>',
      closeTag: '</blockquote>',
      expansionOptions: {
        container: true,
        specials: true // Fall-back if container is disabled.
      }
    },
    // Code block.
    {
      name: 'code',
      // Backtick hex literal \x60 to work arount eslint problem.
      // See https://github.com/palantir/tslint/issues/357.
      openMatch: /^\\?(?:\-{2,}|\x60{2,})$/,
      openTag: '<pre><code>',
      closeTag: '</code></pre>',
      expansionOptions: {
        macros: false,
        specials: true
      }
    },
    // HTML block.
    {
      name: 'html',
      // Must start with  an <! or a block-level element start or end tag.
      // $1 is first line of block.
      /* tslint:disable:max-line-length */
      openMatch: /^(<!.*|(?:<\/?(?:html|head|body|iframe|script|style|address|article|aside|audio|blockquote|canvas|dd|div|dl|fieldset|figcaption|figure|figcaption|footer|form|h1|h2|h3|h4|h5|h6|header|hgroup|hr|img|math|nav|noscript|ol|output|p|pre|section|table|tfoot|td|th|tr|ul|video)(?:[ >].*)?))$/i,
      /* tslint:enable:max-line-length */
      closeMatch: /^$/, // Blank line or EOF.
      openTag: '',
      closeTag: '',
      expansionOptions: {
        macros: true
      },
      filter: function (text: string): string {
        return Options.safeModeFilter(text);
      }
    },
    // Indented paragraph.
    {
      name: 'indented',
      openMatch: /^\\?(\s+.*)$/,  // $1 is first line of block.
      closeMatch: /^$/,           // Blank line or EOF.
      openTag: '<pre><code>',
      closeTag: '</code></pre>',
      expansionOptions: {
        macros: false,
        specials: true
      },
      filter: function (text: string): string {
        // Strip indent from start of each line.
        var first_indent = text.search(/\S/);
        var buffer = text.split('\n');
        for (var i in buffer) {
          // Strip first line indent width or up to first non-space character.
          var indent = buffer[i].search(/\S/);
          if (indent > first_indent) indent = first_indent;
          buffer[i] = buffer[i].slice(indent);
        }
        return buffer.join('\n');
      }
    },
    // Quote paragraph.
    {
      name: 'quote-paragraph',
      openMatch: /^\\?(>.*)$/,      // $1 is first line of block.
      closeMatch: /^$/,             // Blank line or EOF.
      openTag: '<blockquote><p>',
      closeTag: '</p></blockquote>',
      expansionOptions: {
        macros: true,
        spans: true,
        specials: true       // Fall-back if spans is disabled.
      },
      filter: function (text: string): string {
        // Strip leading > from start of each line.
        var buffer = text.split('\n');
        for (var i in buffer) {
          buffer[i] = buffer[i].replace(/^>\s*/, '');
        }
        return buffer.join('\n');
      }
    },
    // Paragraph (lowest priority, cannot be escaped).
    {
      name: 'paragraph',
      openMatch: /^(.*)$/,  // $1 is first line of block.
      closeMatch: /^$/,     // Blank line or EOF.
      openTag: '<p>',
      closeTag: '</p>',
      expansionOptions: {
        macros: true,
        spans: true,
        specials: true       // Fall-back if spans is disabled.
      }
    },
  ];

  // If the next element in the reader is a valid delimited block render it
  // and return true, else return false.
  export function render(reader: Reader, writer: Writer): boolean {
    if (reader.eof()) throw 'premature eof';
    for (var i in defs) {
      var def = defs[i];
      var match = reader.cursor().match(def.openMatch);<commit_msg>Unescape escaped leading '>' in Quote Paragraphs.<commit_after>/// <reference path="references.ts" />

module Rimu.DelimitedBlocks {

  // Multi-line block element definition.
  export interface Definition {
    name?: string;       // Optional unique identifier.
    openMatch: RegExp;   // $1 (if defined) is prepended to block content.
    closeMatch?: RegExp; // $1 (if defined) is appended to block content. If closeMatch is undefined then it must match opening delimiter.
    openTag: string;
    closeTag: string;
    filter?: (text: string, match?: string[], expansionOptions?: ExpansionOptions) => string;
    verify?: (match: string[]) => boolean;  // Additional match verification checks.
    expansionOptions: ExpansionOptions;
  }

  var defs: Definition[] = [
    // Delimited blocks cannot be escaped with a backslash.

    // Macro definition block.
    {
      openMatch: Macros.MACRO_DEF_OPEN,    // $1 is first line of macro.
      closeMatch: Macros.MACRO_DEF_CLOSE,  // $1 is last line of macro.
      openTag: '',
      closeTag: '',
      expansionOptions: {
        macros: true
      },
      filter: function (text: string, match: string[], expansionOptions: ExpansionOptions): string {
        // Set macro.
        // Get the macro name from the match in the first line of the block.
        var name = match[0].match(/^\{([\w\-]+)\}/)[1];
        text = text.replace(/' *\\\n/g, '\'\n');        // Unescape line-continuations.
        text = text.replace(/(' *[\\]+)\\\n/g, '$1\n'); // Unescape escaped line-continuations.
        text = replaceInline(text, expansionOptions);   // Expand macro invocations.
        Macros.setValue(name, text);
        return '';
      }
    },
    // Comment block.
    {
      name: 'comment',
      openMatch: /^\\?\/\*+$/,
      closeMatch: /^\*+\/$/,
      openTag: '',
      closeTag: '',
      expansionOptions: {
        skip: true,
        specials: true // Fall-back if skip is disabled.
      }
    },
    // Division block.
    {
      name: 'division',
      openMatch: /^\\?\.{2,}$/,
      openTag: '<div>',
      closeTag: '</div>',
      expansionOptions: {
        container: true,
        specials: true // Fall-back if container is disabled.
      }
    },
    // Quote block.
    {
      name: 'quote',
      openMatch: /^\\?"{2,}$/,
      openTag: '<blockquote>',
      closeTag: '</blockquote>',
      expansionOptions: {
        container: true,
        specials: true // Fall-back if container is disabled.
      }
    },
    // Code block.
    {
      name: 'code',
      // Backtick hex literal \x60 to work arount eslint problem.
      // See https://github.com/palantir/tslint/issues/357.
      openMatch: /^\\?(?:\-{2,}|\x60{2,})$/,
      openTag: '<pre><code>',
      closeTag: '</code></pre>',
      expansionOptions: {
        macros: false,
        specials: true
      }
    },
    // HTML block.
    {
      name: 'html',
      // Must start with  an <! or a block-level element start or end tag.
      // $1 is first line of block.
      /* tslint:disable:max-line-length */
      openMatch: /^(<!.*|(?:<\/?(?:html|head|body|iframe|script|style|address|article|aside|audio|blockquote|canvas|dd|div|dl|fieldset|figcaption|figure|figcaption|footer|form|h1|h2|h3|h4|h5|h6|header|hgroup|hr|img|math|nav|noscript|ol|output|p|pre|section|table|tfoot|td|th|tr|ul|video)(?:[ >].*)?))$/i,
      /* tslint:enable:max-line-length */
      closeMatch: /^$/, // Blank line or EOF.
      openTag: '',
      closeTag: '',
      expansionOptions: {
        macros: true
      },
      filter: function (text: string): string {
        return Options.safeModeFilter(text);
      }
    },
    // Indented paragraph.
    {
      name: 'indented',
      openMatch: /^\\?(\s+.*)$/,  // $1 is first line of block.
      closeMatch: /^$/,           // Blank line or EOF.
      openTag: '<pre><code>',
      closeTag: '</code></pre>',
      expansionOptions: {
        macros: false,
        specials: true
      },
      filter: function (text: string): string {
        // Strip indent from start of each line.
        var first_indent = text.search(/\S/);
        var buffer = text.split('\n');
        for (var i in buffer) {
          // Strip first line indent width or up to first non-space character.
          var indent = buffer[i].search(/\S/);
          if (indent > first_indent) indent = first_indent;
          buffer[i] = buffer[i].slice(indent);
        }
        return buffer.join('\n');
      }
    },
    // Quote paragraph.
    {
      name: 'quote-paragraph',
      openMatch: /^\\?(>.*)$/,      // $1 is first line of block.
      closeMatch: /^$/,             // Blank line or EOF.
      openTag: '<blockquote><p>',
      closeTag: '</p></blockquote>',
      expansionOptions: {
        macros: true,
        spans: true,
        specials: true       // Fall-back if spans is disabled.
      },
      filter: function (text: string): string {
        // Strip leading > from start of each line and unescape escaped leading >.
        var buffer = text.split('\n');
        for (var i in buffer) {
          buffer[i] = buffer[i].replace(/^>\s*/, '');
          buffer[i] = buffer[i].replace(/^\\>/, '>');
        }
        return buffer.join('\n');
      }
    },
    // Paragraph (lowest priority, cannot be escaped).
    {
      name: 'paragraph',
      openMatch: /^(.*)$/,  // $1 is first line of block.
      closeMatch: /^$/,     // Blank line or EOF.
      openTag: '<p>',
      closeTag: '</p>',
      expansionOptions: {
        macros: true,
        spans: true,
        specials: true       // Fall-back if spans is disabled.
      }
    },
  ];

  // If the next element in the reader is a valid delimited block render it
  // and return true, else return false.
  export function render(reader: Reader, writer: Writer): boolean {
    if (reader.eof()) throw 'premature eof';
    for (var i in defs) {
      var def = defs[i];
      var match = reader.cursor().match(def.openMatch);
{'fraction_non_alphanumeric': 0.16078200050560376, 'fraction_numerical': 0.004718968568298643, 'mean_word_length': 2.0137125444388015, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 59, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2935614', 'n_tokens_mistral': 3678, 'n_tokens_neox': 3660, 'n_words': 992}