<commit_before># -*- coding: utf-8 -*-
from docutils import nodes
from docutils.parsers.rst import Directive
from docutils.parsers.rst import directives
import os
import re
from hashlib import sha1
import traceback

from sphinx.builders import Builder
from sphinx.ext.autodoc import AutodocReporter
from sphinx.util.console import bold

import imgui

try:
    from gen_example import render_snippet
except ImportError:
    render_snippet = None


VISUAL_EXAMPLES_DIR = "visual_examples"

# todo: maybe should be more generic from sphinx conf
SOURCE_DIR = os.path.join(os.path.dirname(__file__))


def flag(argument):
    """Reimplement directives.flag to return True instead of None
    Check for a valid flag option (no argument) and return ``None``.
    (Directive option conversion function.)

    Raise ``ValueError`` if an argument is found.
    """
    if argument and argument.strip():
        raise ValueError('no argument is allowed; "%s" supplied' % argument)
    else:
        return True


def nonnegative_int_list(argument):
    if ',' in argument:
        entries = argument.split(',')
    else:
        entries = argument.split()
    return [directives.nonnegative_int(entry) for entry in entries]


def click_list(argument):
    value = nonnegative_int_list(argument)

    if len(value) != 2:
        ValueError("argument must contain 3 non-negative values")

    return value


class WrapsDirective(Directive):
    has_content = True

    def run(self):
        head = nodes.paragraph()
        head.append(nodes.inline("Wraps API:", "Wraps API: "))

        source = '\n'.join(self.content.data)
        literal_node = nodes.literal_block(source, source)
        literal_node['laguage'] = 'C++'

        return [head, literal_node]


class VisualDirective(Directive):
    has_content = True

    final_argument_whitespace = True
    option_spec = {
        'title': directives.unchanged,
        'width': directives.positive_int,
        'height': directives.positive_int,
        'auto_layout': flag,
        'click': click_list,
    }

    def run(self):
        source = '\n'.join(self.content.data)
        literal = nodes.literal_block(source, source)
        literal['visualnodetype'] = True
        literal['language'] = 'python'

        # docutils document model is insane!
        head1 = nodes.paragraph()
        head1.append(nodes.inline("Example:", "Example: "))

        head2 = nodes.paragraph()
        head2.append(
            nodes.section("foo", nodes.inline("Outputs:", "Outputs: "))
        )

        directive_nodes = [
            head1,
            literal,
            head2,
            self.get_image_node(source)
        ]

        return directive_nodes

    def name_source_snippet(self, source):
        env = self.state.document.settings.env

        if (
            isinstance(self.state.reporter, AutodocReporter) and
            self.state.parent and self.state.parent.parent
        ):
            # If it is generated by autodoc then autogenerate title from
            # the function/method/class signature
            # note: hacky assumption that this is a signature node
            signature_node = self.state.parent.parent.children[0]
            signature = signature_node['names'][0]
            occurence = env.new_serialno(signature)

            name = signature + '_' + str(occurence)
        else:
            # If we could not quess then use explicit title or hexdigest
            name = self.options.get('title', sha1(source).hexdigest())

        return self.phrase_to_filename(name)

    def phrase_to_filename(self, phrase):<commit_msg>dos: fix py2/py3 encoding issue when creating snippet hashes<commit_after># -*- coding: utf-8 -*-
from docutils import nodes
from docutils.parsers.rst import Directive
from docutils.parsers.rst import directives
import os
import re
from hashlib import sha1
import traceback

from sphinx.builders import Builder
from sphinx.ext.autodoc import AutodocReporter
from sphinx.util.console import bold

import imgui

try:
    from gen_example import render_snippet
except ImportError:
    render_snippet = None


VISUAL_EXAMPLES_DIR = "visual_examples"

# todo: maybe should be more generic from sphinx conf
SOURCE_DIR = os.path.join(os.path.dirname(__file__))


def flag(argument):
    """Reimplement directives.flag to return True instead of None
    Check for a valid flag option (no argument) and return ``None``.
    (Directive option conversion function.)

    Raise ``ValueError`` if an argument is found.
    """
    if argument and argument.strip():
        raise ValueError('no argument is allowed; "%s" supplied' % argument)
    else:
        return True


def nonnegative_int_list(argument):
    if ',' in argument:
        entries = argument.split(',')
    else:
        entries = argument.split()
    return [directives.nonnegative_int(entry) for entry in entries]


def click_list(argument):
    value = nonnegative_int_list(argument)

    if len(value) != 2:
        ValueError("argument must contain 3 non-negative values")

    return value


class WrapsDirective(Directive):
    has_content = True

    def run(self):
        head = nodes.paragraph()
        head.append(nodes.inline("Wraps API:", "Wraps API: "))

        source = '\n'.join(self.content.data)
        literal_node = nodes.literal_block(source, source)
        literal_node['laguage'] = 'C++'

        return [head, literal_node]


class VisualDirective(Directive):
    has_content = True

    final_argument_whitespace = True
    option_spec = {
        'title': directives.unchanged,
        'width': directives.positive_int,
        'height': directives.positive_int,
        'auto_layout': flag,
        'click': click_list,
    }

    def run(self):
        source = '\n'.join(self.content.data)
        literal = nodes.literal_block(source, source)
        literal['visualnodetype'] = True
        literal['language'] = 'python'

        # docutils document model is insane!
        head1 = nodes.paragraph()
        head1.append(nodes.inline("Example:", "Example: "))

        head2 = nodes.paragraph()
        head2.append(
            nodes.section("foo", nodes.inline("Outputs:", "Outputs: "))
        )

        directive_nodes = [
            head1,
            literal,
            head2,
            self.get_image_node(source)
        ]

        return directive_nodes

    def name_source_snippet(self, source):
        env = self.state.document.settings.env

        if (
            isinstance(self.state.reporter, AutodocReporter) and
            self.state.parent and self.state.parent.parent
        ):
            # If it is generated by autodoc then autogenerate title from
            # the function/method/class signature
            # note: hacky assumption that this is a signature node
            signature_node = self.state.parent.parent.children[0]
            signature = signature_node['names'][0]
            occurence = env.new_serialno(signature)

            name = signature + '_' + str(occurence)
        else:
            # If we could not quess then use explicit title or hexdigest
            name = self.options.get('title', sha1(source.encode()).hexdigest())

        return self.phrase_to_filename(name)

    def phrase_to_filename(self, phrase):
{'fraction_non_alphanumeric': 0.09761120263591433, 'fraction_numerical': 0.003844041735310269, 'mean_word_length': 2.65529352734571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7126226', 'n_tokens_mistral': 2111, 'n_tokens_neox': 2028, 'n_words': 555}