<commit_before>"""
Render documents as XHTML fragments
"""



from pyth import document
from pyth.format import PythWriter
import base64

from cStringIO import StringIO


_tagNames = {
    'bold': 'strong',
    'italic': 'em',
    'underline': 'u', # ?
    'super': 'sup',
    'sub': 'sub',
}


class XHTMLWriter(PythWriter):

    @classmethod
    def write(klass, document, target=None, cssClasses=True, pretty=False):
        if target is None:
            target = StringIO()

        writer = XHTMLWriter(document, target, cssClasses, pretty)
        final = writer.go()
        final.seek(0)

        # Doesn't work all that well -- appends an <?xml ...> tag,
        # and puts line breaks in unusual places for HTML.
        #if pretty:
        #    content = final.read()
        #    final.seek(0)
        #    from xml.dom.ext import PrettyPrint
        #    from xml.dom.ext.reader.Sax import FromXml
        #    PrettyPrint(FromXml(content), final)
        #    final.seek(0)

        return final


    def __init__(self, doc, target, cssClasses=True, pretty=False):
        self.document = doc
        self.target = target
        self.cssClasses = cssClasses
        self.pretty = pretty
        self.paragraphDispatch = {
            document.List: self._list,
            document.Paragraph: self._paragraph
        }
        self.paragraphContentDispatch = {
            document.Text: self._text,
            document.Image: self._image,
        }


    def go(self):

        self.listLevel = -1

        tag = Tag("div")

        for element in self.document.content:
            handler = self.paragraphDispatch[element.__class__]
            tag.content.extend(handler(element))

        tag.render(self.target)
        return self.target


    def _paragraph(self, paragraph):
        p = Tag("p")
        for item in paragraph.content:
            handler = self.paragraphContentDispatch[item.__class__]
            p.content.append(handler(item))

        if self.pretty:
            return [_prettyBreak, p, _prettyBreak]
        else:
            return [p]


    def _list(self, lst):
        self.listLevel += 1

        ul = Tag("ul")

        if self.cssClasses:
            ul.attrs['class'] = 'pyth_list_%s' % self.listLevel

        for entry in lst.content:
            li = Tag("li")
            for element in entry.content:
                handler = self.paragraphDispatch[element.__class__]
                li.content.extend(handler(element))
            ul.content.append(li)

        self.listLevel -= 1

        return [ul]


    def _text(self, text):
        if 'url' in text.properties:
            tag = Tag("a")
            tag.attrs['href'] = text.properties['url']
        else:
            tag = Tag(None)

        current = tag

        for prop in ('bold', 'italic', 'underline', 'sub', 'super'):
            if prop in text.properties:
                newTag = Tag(_tagNames[prop])
                current.content.append(newTag)
                current = newTag

        current.content.append(u"".join(text.content))

        return tag

    def _image(self, image):
        if image.properties.get(u'pngblip'):
            tag = Tag("img")
            image_data = bytearray.fromhex(image.content[0])<commit_msg>Better sublist handling in xhtml writer<commit_after>"""
Render documents as XHTML fragments
"""



from pyth import document
from pyth.format import PythWriter
import base64

from cStringIO import StringIO


_tagNames = {
    'bold': 'strong',
    'italic': 'em',
    'underline': 'u', # ?
    'super': 'sup',
    'sub': 'sub',
}


class XHTMLWriter(PythWriter):

    @classmethod
    def write(klass, document, target=None, cssClasses=True, pretty=False):
        if target is None:
            target = StringIO()

        writer = XHTMLWriter(document, target, cssClasses, pretty)
        final = writer.go()
        final.seek(0)

        # Doesn't work all that well -- appends an <?xml ...> tag,
        # and puts line breaks in unusual places for HTML.
        #if pretty:
        #    content = final.read()
        #    final.seek(0)
        #    from xml.dom.ext import PrettyPrint
        #    from xml.dom.ext.reader.Sax import FromXml
        #    PrettyPrint(FromXml(content), final)
        #    final.seek(0)

        return final


    def __init__(self, doc, target, cssClasses=True, pretty=False):
        self.document = doc
        self.target = target
        self.cssClasses = cssClasses
        self.pretty = pretty
        self.paragraphDispatch = {
            document.List: self._list,
            document.Paragraph: self._paragraph
        }
        self.paragraphContentDispatch = {
            document.Text: self._text,
            document.Image: self._image,
        }


    def go(self):

        self.listLevel = -1

        tag = Tag("div")

        for element in self.document.content:
            handler = self.paragraphDispatch[element.__class__]
            tag.content.extend(handler(element))

        tag.render(self.target)
        return self.target


    def _paragraph(self, paragraph):
        p = Tag("p")
        for item in paragraph.content:
            handler = self.paragraphContentDispatch[item.__class__]
            p.content.append(handler(item))

        if self.pretty:
            return [_prettyBreak, p, _prettyBreak]
        else:
            return [p]


    def _list(self, lst):
        self.listLevel += 1

        ul = Tag("ul")

        if self.cssClasses:
            ul.attrs['class'] = 'pyth_list_%s' % self.listLevel

        last_li = None
        for entry in lst.content:
            li = Tag("li")
            for element in entry.content:
                # in practice list elements always have only one content child?
                handler = self.paragraphDispatch[element.__class__]
                if handler == self._list:
                    # this is a sublist, so we shouldn't create an empty li, but rather append ul to prior li.
                    # Lists can't be immediately sublisted (e.g. there must be at least something at outer level)
                    # but if that is not the case the last_li will be None and next line will bomb out, which is a
                    # useful implicit assertion
                    last_li.content.extend(handler(element))
                else:
                    li.content.extend(handler(element))
                    last_li = li
            if li.content:  # li might be empty..
                ul.content.append(li)

        self.listLevel -= 1

        return [ul]


    def _text(self, text):
        if 'url' in text.properties:
            tag = Tag("a")
            tag.attrs['href'] = text.properties['url']
        else:
            tag = Tag(None)

        current = tag

        for prop in ('bold', 'italic', 'underline', 'sub', 'super'):
            if prop in text.properties:
                newTag = Tag(_tagNames[prop])
                current.content.append(newTag)
                current = newTag

        current.content.append(u"".join(text.content))

        return tag

    def _image(self, image):
        if image.properties.get(u'pngblip'):
            tag = Tag("img")
            image_data = bytearray.fromhex(image.content[0])
{'fraction_non_alphanumeric': 0.1047998899738688, 'fraction_numerical': 0.002475587952138633, 'mean_word_length': 1.8088064889918887, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2334301', 'n_tokens_mistral': 2080, 'n_tokens_neox': 1971, 'n_words': 538}