<commit_before># -*- coding: utf-8 -*-
import os.path
import posixpath
import re
import urllib

from docutils import nodes
from sphinx import addnodes, util
from sphinx.locale import admonitionlabels


def _parents(node):
    while node.parent:
        node = node.parent
        yield node

class BootstrapTranslator(nodes.NodeVisitor, object):
    head_prefix = 'head_prefix'
    head = 'head'
    stylesheet = 'stylesheet'
    body_prefix = 'body_prefix'
    body_pre_docinfo = 'body_pre_docinfo'
    docinfo = 'docinfo'
    body_suffix = 'body_suffix'
    subtitle = 'subtitle'
    header = 'header'
    footer = 'footer'
    html_prolog = 'html_prolog'
    html_head = 'html_head'
    html_title = 'html_title'
    html_subtitle = 'html_subtitle'

    # <meta> tags
    meta = [
        '<meta http-equiv="X-UA-Compatible" content="IE=edge">',
        '<meta name="viewport" content="width=device-width, initial-scale=1">'
    ]

    def __init__(self, builder, document):
        super(BootstrapTranslator, self).__init__(document)
        self.builder = builder
        self.body = []
        self.fragment = self.body
        self.html_body = self.body
        # document title
        self.title = []
        self.start_document_title = 0
        self.first_title = False

        self.context = []
        self.section_level = 0

        self.highlightlang = self.highlightlang_base = self.builder.config.highlight_language
        self.highlightopts = getattr(builder.config, 'highlight_options', {})

        self.first_param = 1
        self.optional_param_level = 0
        self.required_params_left = 0
        self.param_separator = ','

    def encode(self, text):
        return unicode(text).translate({
            ord('&'): u'&amp;',
            ord('<'): u'&lt;',
            ord('"'): u'&quot;',
            ord('>'): u'&gt;',
            0xa0: u'&nbsp;'
        })

    def starttag(self, node, tagname, **attributes):
        tagname = unicode(tagname).lower()

        # extract generic attributes
        attrs = {name.lower(): value for name, value in attributes.iteritems()}
        attrs.update(
            (name, value) for name, value in node.attributes.iteritems()
            if name.startswith('data-')
        )

        prefix = []
        postfix = []

        # handle possibly multiple ids
        assert 'id' not in attrs, "starttag can't be passed a single id attribute, use a list of ids"
        ids = node.get('ids', []) + attrs.pop('ids', [])
        if ids:
            _ids = iter(ids)
            attrs['id'] = next(_ids)
            postfix.extend(u'<i id="{}"></i>'.format(_id) for _id in _ids)

        # set CSS class
        classes = set(node.get('classes', []) + attrs.pop('class', '').split())
        if classes:
            attrs['class'] = u' '.join(classes)

        return u'{prefix}<{tag} {attrs}>{postfix}'.format(
            prefix=u''.join(prefix),
            tag=tagname,
            attrs=u' '.join(u'{}="{}"'.format(name, self.attval(value))
                            for name,  value in attrs.iteritems()),
            postfix=u''.join(postfix),
        )
    # only "space characters" SPACE, CHARACTER TABULATION, LINE FEED,
    # FORM FEED and CARRIAGE RETURN should be collapsed, not al White_Space
    def attval(self, value, whitespace=re.compile(u'[ \t\n\f\r]')):
        return self.encode(whitespace.sub(u' ', unicode(value)))

    def astext(self):
        return u''.join(self.body)

    def unknown_visit(self, node):
        print "unknown node", node.__class__.__name__
        self.body.append(u'[UNKNOWN NODE {}]'.format(node.__class__.__name__))
        raise nodes.SkipNode

    def visit_highlightlang(self, node):
        self.highlightlang = node['lang']
    def depart_highlightlang(self, node):
        pass

    def visit_document(self, node):
        self.first_title = True
    def depart_document(self, node):
        pass

    def visit_section(self, node):
        # close "parent" or preceding section, unless this is the opening of
        # the first section
        if self.section_level:
            self.body.append(u'</section>')
        self.section_level += 1

        self.body.append(self.starttag(node, 'section'))
    def depart_section(self, node):
        self.section_level -= 1
        # close last section of document
        if not self.section_level:
            self.body.append(u'</section>')

    def is_compact_paragraph(self, node):
        parent = node.parent
        if isinstance(parent, (nodes.document, nodes.compound,
                               addnodes.desc_content,
                               addnodes.versionmodified)):
            # Never compact paragraphs in document or compound.
            return False
        first = isinstance(node.parent[0], nodes.label)
        for child in parent.children[first:]:
            # only first paragraph can be compact
            if isinstance(child, nodes.Invisible):
                continue
            if child is node:
                break
            return False
        parent_length = len([
            1 for n in parent
            if not isinstance(n, (nodes.Invisible, nodes.label))
        ])
        return parent_length == 1

    def visit_paragraph(self, node):
        if self.is_compact_paragraph(node):
            self.context.append(u'')
            return<commit_msg>[FIX] doc: prevent compacting paragraphs with custom classes<commit_after># -*- coding: utf-8 -*-
import os.path
import posixpath
import re
import urllib

from docutils import nodes
from sphinx import addnodes, util
from sphinx.locale import admonitionlabels


def _parents(node):
    while node.parent:
        node = node.parent
        yield node

class BootstrapTranslator(nodes.NodeVisitor, object):
    head_prefix = 'head_prefix'
    head = 'head'
    stylesheet = 'stylesheet'
    body_prefix = 'body_prefix'
    body_pre_docinfo = 'body_pre_docinfo'
    docinfo = 'docinfo'
    body_suffix = 'body_suffix'
    subtitle = 'subtitle'
    header = 'header'
    footer = 'footer'
    html_prolog = 'html_prolog'
    html_head = 'html_head'
    html_title = 'html_title'
    html_subtitle = 'html_subtitle'

    # <meta> tags
    meta = [
        '<meta http-equiv="X-UA-Compatible" content="IE=edge">',
        '<meta name="viewport" content="width=device-width, initial-scale=1">'
    ]

    def __init__(self, builder, document):
        super(BootstrapTranslator, self).__init__(document)
        self.builder = builder
        self.body = []
        self.fragment = self.body
        self.html_body = self.body
        # document title
        self.title = []
        self.start_document_title = 0
        self.first_title = False

        self.context = []
        self.section_level = 0

        self.highlightlang = self.highlightlang_base = self.builder.config.highlight_language
        self.highlightopts = getattr(builder.config, 'highlight_options', {})

        self.first_param = 1
        self.optional_param_level = 0
        self.required_params_left = 0
        self.param_separator = ','

    def encode(self, text):
        return unicode(text).translate({
            ord('&'): u'&amp;',
            ord('<'): u'&lt;',
            ord('"'): u'&quot;',
            ord('>'): u'&gt;',
            0xa0: u'&nbsp;'
        })

    def starttag(self, node, tagname, **attributes):
        tagname = unicode(tagname).lower()

        # extract generic attributes
        attrs = {name.lower(): value for name, value in attributes.iteritems()}
        attrs.update(
            (name, value) for name, value in node.attributes.iteritems()
            if name.startswith('data-')
        )

        prefix = []
        postfix = []

        # handle possibly multiple ids
        assert 'id' not in attrs, "starttag can't be passed a single id attribute, use a list of ids"
        ids = node.get('ids', []) + attrs.pop('ids', [])
        if ids:
            _ids = iter(ids)
            attrs['id'] = next(_ids)
            postfix.extend(u'<i id="{}"></i>'.format(_id) for _id in _ids)

        # set CSS class
        classes = set(node.get('classes', []) + attrs.pop('class', '').split())
        if classes:
            attrs['class'] = u' '.join(classes)

        return u'{prefix}<{tag} {attrs}>{postfix}'.format(
            prefix=u''.join(prefix),
            tag=tagname,
            attrs=u' '.join(u'{}="{}"'.format(name, self.attval(value))
                            for name,  value in attrs.iteritems()),
            postfix=u''.join(postfix),
        )
    # only "space characters" SPACE, CHARACTER TABULATION, LINE FEED,
    # FORM FEED and CARRIAGE RETURN should be collapsed, not al White_Space
    def attval(self, value, whitespace=re.compile(u'[ \t\n\f\r]')):
        return self.encode(whitespace.sub(u' ', unicode(value)))

    def astext(self):
        return u''.join(self.body)

    def unknown_visit(self, node):
        print "unknown node", node.__class__.__name__
        self.body.append(u'[UNKNOWN NODE {}]'.format(node.__class__.__name__))
        raise nodes.SkipNode

    def visit_highlightlang(self, node):
        self.highlightlang = node['lang']
    def depart_highlightlang(self, node):
        pass

    def visit_document(self, node):
        self.first_title = True
    def depart_document(self, node):
        pass

    def visit_section(self, node):
        # close "parent" or preceding section, unless this is the opening of
        # the first section
        if self.section_level:
            self.body.append(u'</section>')
        self.section_level += 1

        self.body.append(self.starttag(node, 'section'))
    def depart_section(self, node):
        self.section_level -= 1
        # close last section of document
        if not self.section_level:
            self.body.append(u'</section>')

    def is_compact_paragraph(self, node):
        parent = node.parent
        if isinstance(parent, (nodes.document, nodes.compound,
                               addnodes.desc_content,
                               addnodes.versionmodified)):
            # Never compact paragraphs in document or compound.
            return False

        for key, value in node.attlist():
            # we can ignore a few specific classes, all other non-default
            # attributes require that a <p> node remains
            if key != 'classes' or value not in ([], ['first'], ['last'], ['first', 'last']):
                return False

        first = isinstance(node.parent[0], nodes.label)
        for child in parent.children[first:]:
            # only first paragraph can be compact
            if isinstance(child, nodes.Invisible):
                continue
            if child is node:
                break
            return False
        parent_length = len([
            1 for n in parent
            if not isinstance(n, (nodes.Invisible, nodes.label))
        ])
        return parent_length == 1

    def visit_paragraph(self, node):
        if self.is_compact_paragraph(node):
            self.context.append(u'')
            return
{'fraction_non_alphanumeric': 0.11987751058272539, 'fraction_numerical': 0.002521840943889039, 'mean_word_length': 2.1752931083786105, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1864216', 'n_tokens_mistral': 3297, 'n_tokens_neox': 3191, 'n_words': 805}