<commit_before>from .base_renderer import *
import re
import markdown


@renderer
class MarkdownRenderer(MarkupRenderer):
    FILENAME_PATTERN_RE = re.compile(r'\.(md|mmd|mkdn?|mdwn|mdown|markdown|litcoffee)$')
    YAML_FRONTMATTER_RE = re.compile(r'\A---\s*\n.*?\n?^---\s*$\n?', re.DOTALL | re.MULTILINE)
    MARKDOWN_SYNTAX_RE = re.compile(r'^text\.html\.markdown\S*')

    def load_settings(self, renderer_options, global_setting):
        super(MarkdownRenderer, self).load_settings(renderer_options, global_setting)
        if 'extensions' in renderer_options:
            extensions = renderer_options['extensions']
        else:
            # Fallback to the default GFM style
            extensions = ['tables', 'strikeout', 'fenced_code', 'codehilite']
        extensions = set(extensions)
        if global_setting.mathjax_enabled:
            if 'mathjax' not in extensions:
                extensions.add('mathjax')
        if 'smartypants' in extensions:
            extensions.remove('smartypants')
            extensions.add('smarty')
        self.extensions = list(extensions)<commit_msg>Do not guess language by default when codehilite is enabled<commit_after>from .base_renderer import *
import re
import markdown


@renderer
class MarkdownRenderer(MarkupRenderer):
    FILENAME_PATTERN_RE = re.compile(r'\.(md|mmd|mkdn?|mdwn|mdown|markdown|litcoffee)$')
    YAML_FRONTMATTER_RE = re.compile(r'\A---\s*\n.*?\n?^---\s*$\n?', re.DOTALL | re.MULTILINE)
    MARKDOWN_SYNTAX_RE = re.compile(r'^text\.html\.markdown\S*')

    def load_settings(self, renderer_options, global_setting):
        super(MarkdownRenderer, self).load_settings(renderer_options, global_setting)
        if 'extensions' in renderer_options:
            extensions = renderer_options['extensions']
        else:
            # Fallback to the default GFM style
            extensions = ['tables', 'strikeout', 'fenced_code', 'codehilite']
        extensions = set(extensions)
        if global_setting.mathjax_enabled:
            if 'mathjax' not in extensions:
                extensions.add('mathjax')
        if 'smartypants' in extensions:
            extensions.remove('smartypants')
            extensions.add('smarty')
        if 'codehilite' in extensions:
            extensions.remove('codehilite')
            extensions.add('codehilite(guess_lang=False)')
        self.extensions = list(extensions)
{'fraction_non_alphanumeric': 0.12103505843071787, 'fraction_numerical': 0.0, 'mean_word_length': 3.132758620689655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6334281', 'n_tokens_mistral': 751, 'n_tokens_neox': 727, 'n_words': 136}