<commit_before>const unified = require('unified');
const visit = require('unist-util-visit');
const toHast = require('mdast-util-to-hast');
const raw = require('hast-util-raw');
const toHtml = require('hast-util-to-html');
const isEmpty = require('lodash/isEmpty');
const hastToMdast = require('hast-util-to-mdast');
const remarkStringify = require('remark-stringify');
const remarkParse = require('remark-parse');
const { text } = require('mdast-builder');
const parseEntities = require('parse-entities');
const find = require('unist-util-find');
const h = require('hastscript');
const phrasing = require('hast-util-phrasing');

const blankLine = { type: 'text', value: '\n\n' };

/* Currently the challenge parser behaves differently depending on whether a
section starts with an empty line or not.  If it does not, the parser interprets
single new-line (\n) characters as paragraph breaks.  It also does not parse
markdown syntax (such as `).  This makes formatting challenges harder than it
needs to be, since normal markdown rules do not apply (some of the time!)

For example
<section id='instructions'>
Sentence1.
Sentence2 `var x = 'y'`.
...

becomes


Sentence1.

Sentence2 `var x = 'y'`.


in the challenge, but should become


Sentence1. Sentence2 <code>var x = 'y'</code>.

---

This file converts the instructions and descriptions.  After this there will be
no need to handle the case where the first line is not empty and markdown syntax
will alway work.  The linter can check that the first blank line exists.
*/

var parser = unified().use(remarkParse);
var mdCompiler = unified().use(remarkStringify);

function stringifyMd(mdast) {
  return mdCompiler.stringify(mdast);
}

function parseMd(text) {
  return parser.parse(text);
}

function escapeMd(hastNode) {
  // defensive copy because stringify mutates objects its called on...
  hastNode = { ...hastNode };
  // These are added by getParagraphs so must not be touched
  if (hastNode.value === '\n\n') return hastNode;
  // A trailing space gets converted to \n, because hastToMdast will be told
  // told this is a paragraph, which it isn't
  const trailingSpace = /\s/.test(hastNode.value[hastNode.value.length - 1]);
  // leading spaces also get ignored, but need to be added in, since they can
  // be following html elements.
  const leadingSpace = /\s/.test(hastNode.value[0]);
  // fake a hast tree.  Is there a less hacky approach?<commit_msg>fix(tools): message typo in insert-spaces.js (#40614)<commit_after>const unified = require('unified');
const visit = require('unist-util-visit');
const toHast = require('mdast-util-to-hast');
const raw = require('hast-util-raw');
const toHtml = require('hast-util-to-html');
const isEmpty = require('lodash/isEmpty');
const hastToMdast = require('hast-util-to-mdast');
const remarkStringify = require('remark-stringify');
const remarkParse = require('remark-parse');
const { text } = require('mdast-builder');
const parseEntities = require('parse-entities');
const find = require('unist-util-find');
const h = require('hastscript');
const phrasing = require('hast-util-phrasing');

const blankLine = { type: 'text', value: '\n\n' };

/* Currently the challenge parser behaves differently depending on whether a
section starts with an empty line or not.  If it does not, the parser interprets
single new-line (\n) characters as paragraph breaks.  It also does not parse
markdown syntax (such as `).  This makes formatting challenges harder than it
needs to be, since normal markdown rules do not apply (some of the time!)

For example
<section id='instructions'>
Sentence1.
Sentence2 `var x = 'y'`.
...

becomes


Sentence1.

Sentence2 `var x = 'y'`.


in the challenge, but should become


Sentence1. Sentence2 <code>var x = 'y'</code>.

---

This file converts the instructions and descriptions.  After this there will be
no need to handle the case where the first line is not empty and markdown syntax
will always work.  The linter can check that the first blank line exists.
*/

var parser = unified().use(remarkParse);
var mdCompiler = unified().use(remarkStringify);

function stringifyMd(mdast) {
  return mdCompiler.stringify(mdast);
}

function parseMd(text) {
  return parser.parse(text);
}

function escapeMd(hastNode) {
  // defensive copy because stringify mutates objects its called on...
  hastNode = { ...hastNode };
  // These are added by getParagraphs so must not be touched
  if (hastNode.value === '\n\n') return hastNode;
  // A trailing space gets converted to \n, because hastToMdast will be told
  // told this is a paragraph, which it isn't
  const trailingSpace = /\s/.test(hastNode.value[hastNode.value.length - 1]);
  // leading spaces also get ignored, but need to be added in, since they can
  // be following html elements.
  const leadingSpace = /\s/.test(hastNode.value[0]);
  // fake a hast tree.  Is there a less hacky approach?
{'fraction_non_alphanumeric': 0.11791197543500512, 'fraction_numerical': 0.004298874104401228, 'mean_word_length': 4.265086206896552, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '412433', 'n_tokens_mistral': 1479, 'n_tokens_neox': 1442, 'n_words': 552}