<commit_before># Directive for embedding code examples

from docutils import nodes

from sphinx.directives.code import CodeBlock
from sphinx.util.compat import Directive, make_admonition


class CodeSnippet(Directive):    # Arguments: title, language
    """
    A code snippet in a language.
    """

    has_content = True
    required_arguments = 2
    optional_arguments = 0
    final_argument_whitespace = False

    def run(self):
        env = self.state.document.settings.env

        key = self.arguments[0]
        lang = self.arguments[1]
        lang_pretty = lang.capitalize()
        lang_hilight = lang

        # Custom pretty names and hilight schemes from config file
        for (lkey, lname, lhilight) in env.config.snippet_langs:
            if lkey == lang:
                lang_pretty = lname
                lang_hilight = lhilight
                break

        node = SingleSnippetNode()
        node['key'] = key
        node['language'] = lang
        node['language-pretty'] = lang_pretty
                        # TODO: don't require same language terms in directive

        code = u'\n'.join(self.content)
        literal = nodes.literal_block(code, code)
        literal['language'] = lang_hilight     # For syntax hilighting.

        node.append(literal)    # Wrap the code block in our SingleSnippetNode

        # Register in the environment for use in the resolve stage
        if not hasattr(env, 'snippets_all'):
            env.snippets_all = []

        env.snippets_all.append({
            'docname': env.docname,
            'content': node<commit_msg>Remove obsolete todo<commit_after># Directive for embedding code examples

from docutils import nodes

from sphinx.directives.code import CodeBlock
from sphinx.util.compat import Directive, make_admonition


class CodeSnippet(Directive):    # Arguments: title, language
    """
    A code snippet in a language.
    """

    has_content = True
    required_arguments = 2
    optional_arguments = 0
    final_argument_whitespace = False

    def run(self):
        env = self.state.document.settings.env

        key = self.arguments[0]
        lang = self.arguments[1]
        lang_pretty = lang.capitalize()
        lang_hilight = lang

        # Custom pretty names and hilight schemes from config file
        for (lkey, lname, lhilight) in env.config.snippet_langs:
            if lkey == lang:
                lang_pretty = lname
                lang_hilight = lhilight
                break

        node = SingleSnippetNode()
        node['key'] = key
        node['language'] = lang
        node['language-pretty'] = lang_pretty

        code = u'\n'.join(self.content)
        literal = nodes.literal_block(code, code)
        literal['language'] = lang_hilight     # For syntax hilighting.

        node.append(literal)    # Wrap the code block in our SingleSnippetNode

        # Register in the environment for use in the resolve stage
        if not hasattr(env, 'snippets_all'):
            env.snippets_all = []

        env.snippets_all.append({
            'docname': env.docname,
            'content': node
{'fraction_non_alphanumeric': 0.08167841127482384, 'fraction_numerical': 0.0025624599615631004, 'mean_word_length': 2.2262396694214877, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '898358', 'n_tokens_mistral': 914, 'n_tokens_neox': 843, 'n_words': 247}