<commit_before>import re
from typing import Any, Iterable, Mapping

from markdown_it import MarkdownIt

import mdformat.plugins


def is_md_equal(
    md1: str,
    md2: str,
    options: Mapping[str, Any],
    *,
    extensions: Iterable[str] = (),
    codeformatters: Iterable[str] = (),
) -> bool:
    """Check if two Markdown produce the same HTML.

    Renders HTML from both Markdown strings, strips whitespace and
    checks equality. Note that this is not a perfect solution, as there
    can be meaningful whitespace in HTML, e.g. in a <code> block.
    """
    html_texts = {}
    mdit = MarkdownIt()
    mdit.options["mdformat"] = options
    for extension in extensions:
        mdformat.plugins.PARSER_EXTENSIONS[extension].update_mdit(mdit)
    for key, text in [("md1", md1), ("md2", md2)]:
        html = mdit.render(text)
        html = re.sub(r"\s+", "", html)
        for codeclass in codeformatters:
            html = re.sub(rf'<codeclass="language-{codeclass}">.*</pre>', "", html)
        html_texts[key] = html

    return html_texts["md1"] == html_texts["md2"]<commit_msg>Make code block stripping regex make sense (#46)<commit_after>import re
from typing import Any, Iterable, Mapping

from markdown_it import MarkdownIt

import mdformat.plugins


def is_md_equal(
    md1: str,
    md2: str,
    options: Mapping[str, Any],
    *,
    extensions: Iterable[str] = (),
    codeformatters: Iterable[str] = (),
) -> bool:
    """Check if two Markdown produce the same HTML.

    Renders HTML from both Markdown strings, strips whitespace and
    checks equality. Note that this is not a perfect solution, as there
    can be meaningful whitespace in HTML, e.g. in a <code> block.
    """
    html_texts = {}
    mdit = MarkdownIt()
    mdit.options["mdformat"] = options
    for extension in extensions:
        mdformat.plugins.PARSER_EXTENSIONS[extension].update_mdit(mdit)
    for key, text in [("md1", md1), ("md2", md2)]:
        html = mdit.render(text)
        for codeclass in codeformatters:
            html = re.sub(f'<code class="language-{codeclass}">.*</code>', "", html)
        html = re.sub(r"\s+", "", html)
        html_texts[key] = html

    return html_texts["md1"] == html_texts["md2"]
{'fraction_non_alphanumeric': 0.12690582959641256, 'fraction_numerical': 0.008071748878923767, 'mean_word_length': 3.1391465677179964, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4847127', 'n_tokens_mistral': 731, 'n_tokens_neox': 714, 'n_words': 213}