<commit_before>import * as remarkParse from 'remark-parse';

function embedTokenizer() {
  function locator(value: string, fromIndex: number) {
    return value.indexOf('%[', fromIndex);
  }

  function inlineTokenizer(eat: remarkParse.Eat, value: string, silent: true): boolean | void {
    // Custom Markdown to Embed a Discussion
    // e.g. given a discussion_id of 123: %[](#123)
    const regex = new RegExp(/%\[\]\(#(\d+)\)\n*/);
    const result = regex.exec(value);

    if (silent) {
      return silent;
    }

    if (!result || result.index !== 0) {
      return;
    }

    const [matched, embed, reference] = result;

    return eat(matched)({
      children: [
        {type: 'text', value: embed},
      ],
      data: {
        discussion_id: embed,
      },
      reference,
      type: 'embed',
    });
  }

  const Parser = this.Parser;
  const inlineTokenizers = Parser.prototype.inlineTokenizers;
  const inlineMethods = Parser.prototype.inlineMethods;

  // Inject inlineTokenizer
  inlineTokenizer.locator = locator;
  inlineTokenizers.embed = inlineTokenizer;
  inlineMethods.splice(0, 0, 'embed');
}

module.exports = embedTokenizer;<commit_msg>Clean up tokenizer a bit<commit_after>import { Eat } from 'remark-parse';
import { Node } from 'unist';

function embedTokenizer() {
  function locator(value: string, fromIndex: number) {
    return value.indexOf('%[', fromIndex);
  }

  function inlineTokenizer(eat: Eat, value: string, silent?: true): Node | boolean | void {
    // Custom Markdown to Embed a Discussion
    // e.g. given a discussion_id of 123: %[](#123)
    const regex = new RegExp(/%\[\]\(#(\d+)\)\n*/);
    const result = regex.exec(value);

    if (silent) {
      return true;
    }

    if (!result || result.index !== 0) {
      return;
    }

    const [matched, embed, reference] = result;

    return eat(matched)({
      children: [
        {type: 'text', value: embed},
      ],
      data: {
        discussion_id: embed,
      },
      reference,
      type: 'embed',
    });
  }

  // Ideally 'Parser' here should be typed like Parser from 'remark-parse', but the provided types appear wonky --
  // they causes issues with the inlineTokenizer definition below, so we're gonna leave it as an implicit 'any' for now.
  const Parser = this.Parser;
  const inlineTokenizers = Parser.prototype.inlineTokenizers;
  const inlineMethods = Parser.prototype.inlineMethods;

  // Inject inlineTokenizer
  inlineTokenizer.locator = locator;
  inlineTokenizers.embed = inlineTokenizer;
  inlineMethods.unshift('embed');
}

module.exports = embedTokenizer;
{'fraction_non_alphanumeric': 0.13379469434832755, 'fraction_numerical': 0.006151480199923107, 'mean_word_length': 3.015432098765432, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6961647', 'n_tokens_mistral': 822, 'n_tokens_neox': 781, 'n_words': 223}