<commit_before>from base_renderer import *
import re
import markdown


@renderer
class Renderer(MarkupRenderer):
    filename_pattern = re.compile(r'\.(md|mkdn?|mdwn|mdown|markdown)$')

    @classmethod
    def is_enabled(cls, filename, syntax):
        if syntax == "text.html.markdown":
            return True
        return cls.filename_pattern.search(filename)

    def render(self, text, **kwargs):
        return markdown.markdown(text, ['codehilite'], tab_length=2, output_format='html5')
<commit_msg>Added fenced code blocks to markdown<commit_after>from base_renderer import *
import re
import markdown


@renderer
class Renderer(MarkupRenderer):
    filename_pattern = re.compile(r'\.(md|mkdn?|mdwn|mdown|markdown)$')

    @classmethod
    def is_enabled(cls, filename, syntax):
        if syntax == "text.html.markdown":
            return True
        return cls.filename_pattern.search(filename)

    def render(self, text, **kwargs):
        return markdown.markdown(text, tab_length=2, output_format='html5',
            extensions=[
                'fenced_code',
                'codehilite',
            ]
        )

{'fraction_non_alphanumeric': 0.11365638766519824, 'fraction_numerical': 0.003524229074889868, 'mean_word_length': 3.145985401459854, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2755416', 'n_tokens_mistral': 344, 'n_tokens_neox': 335, 'n_words': 73}