<commit_before>import lxml.etree
import zipfile
from StringIO import StringIO
import urllib
from genshi.template import MarkupTemplate
from pyjon.utils import get_secure_filename
import os
import decimal

GENSHI_URI = 'http://genshi.edgewall.org/'
PY3O_URI = 'http://py3o.org/'

def move_siblings(start, end, new_):
    old_ = start.getparent()

    # copy any tail we find
    if start.tail:
        new_.text = start.tail

    # get all siblings
    for node in start.itersiblings():
        if not node is end:
            # and stuff them in our new node
            new_.append(node)
        else:
            # if this is already the end boundary, then we are done
            break

    # replace start boundary with new node
    old_.replace(start, new_)
    # remove ending boundary
    old_.remove(end)

class Template(object):
    templated_files = ['content.xml', 'styles.xml']

    def __init__(self, template, outfile):
        """A template object exposes the API to render it to an OpenOffice
        document.

        @param template: a py3o template file. ie: a OpenDocument with the
        proper py3o markups
        @type template: a string representing the full path name to a py3o
        template file.

        @param outfile: the desired file name for the resulting ODT document
        @type outfile: a string representing the full filename for output
        """
        self.template = template
        self.outputfilename = outfile
        self.infile = zipfile.ZipFile(self.template, 'r')
        
        self.content_trees = [lxml.etree.parse(StringIO(self.infile.read(filename)))
                              for filename in self.templated_files]
        self.tree_roots = [tree.getroot() for tree in self.content_trees]
        
#        self.py3ocontent = lxml.etree.parse(StringIO(self.infile.read("content.xml")))
#        self.py3oroot = self.py3ocontent.getroot()
        self.__prepare_namespaces()

    def __prepare_namespaces(self):
        """create proper namespaces for our document
        """
        # create needed namespaces
        self.namespaces = dict(
            text="urn:text",
            draw="urn:draw",
            table="urn:table",
            office="urn:office",
            xlink="urn:xlink",
            svg="urn:svg",
            )

        # copy namespaces from original docs
        for tree_root in self.tree_roots:
            self.namespaces.update(tree_root.nsmap)

        # remove any "root" namespace as lxml.xpath do not support them
        self.namespaces.pop(None, None)

        # declare the genshi namespace
        self.namespaces['py'] = GENSHI_URI
        # declare our own namespace
        self.namespaces['py3o'] = PY3O_URI

    def __handle_instructions(self):
        # find all links that have a py3o
        xpath_expr = "//text:a[starts-with(@xlink:href, 'py3o://')]"

        opened_starts = list()
        starting_tags = list()
        closing_tags = dict()
        
        for content_tree in self.content_trees:
            for link in content_tree.xpath(xpath_expr, namespaces=self.namespaces):
                py3o_statement = urllib.unquote(link.attrib['{%s}href' % self.namespaces['xlink']])
                # remove the py3o://
                py3o_base = py3o_statement[7:]
    
                if not py3o_base.startswith("/"):
                    opened_starts.append((content_tree, link))
                    starting_tags.append((content_tree, link, py3o_base))
    
                else:
                    closing_tags[id(opened_starts.pop()[1])] = (content_tree, link)

        return starting_tags, closing_tags

    def __handle_link(self, content_tree, link, py3o_base, closing_link):
        """transform a py3o link into a proper Genshi statement
        rebase a py3o link a a proper place in the tree
        to be ready for Genshi replacement
        """
        if not link.text == py3o_base:
            msg = "url and text do not match in '%s'" % link.text
            raise ValueError(msg)

        if link.getparent().getparent().tag == "{%s}table-cell" % self.namespaces['table']:
            # we are in a table
            opening_paragraph = link.getparent()
            opening_cell = opening_paragraph.getparent()
            opening_row = opening_cell.getparent()

            # same for closing
            closing_paragraph = closing_link.getparent()
            closing_cell = closing_paragraph.getparent()
            closing_row = closing_cell.getparent()

        elif link.getparent().tag == "{%s}p" % self.namespaces['text']:
            # we are in a text paragraph
            opening_row = link.getparent()
            closing_row = closing_link.getparent()

        else:
            raise NotImplementedError("We handle urls in tables or text paragraph only")

        # max split is one
        instruction, instruction_value = py3o_base.split("=", 1)
        instruction_value = instruction_value.strip('"')

        attribs = dict()
        attribs['{%s}strip' % GENSHI_URI] = 'True'
        attribs['{%s}%s' % (GENSHI_URI, instruction)] = instruction_value

        genshi_node = lxml.etree.Element('span',<commit_msg>Fixed error caused by using documents created in the recent versions of OpenOffice.<commit_after>import lxml.etree
import zipfile
from StringIO import StringIO
import urllib
from genshi.template import MarkupTemplate
from pyjon.utils import get_secure_filename
import os
import decimal

GENSHI_URI = 'http://genshi.edgewall.org/'
PY3O_URI = 'http://py3o.org/'

def move_siblings(start, end, new_):
    old_ = start.getparent()

    # copy any tail we find
    if start.tail:
        new_.text = start.tail

    # get all siblings
    for node in start.itersiblings():
        if not node is end:
            # and stuff them in our new node
            new_.append(node)
        else:
            # if this is already the end boundary, then we are done
            break

    # replace start boundary with new node
    old_.replace(start, new_)
    # remove ending boundary
    old_.remove(end)

class Template(object):
    templated_files = ['content.xml', 'styles.xml']

    def __init__(self, template, outfile):
        """A template object exposes the API to render it to an OpenOffice
        document.

        @param template: a py3o template file. ie: a OpenDocument with the
        proper py3o markups
        @type template: a string representing the full path name to a py3o
        template file.

        @param outfile: the desired file name for the resulting ODT document
        @type outfile: a string representing the full filename for output
        """
        self.template = template
        self.outputfilename = outfile
        self.infile = zipfile.ZipFile(self.template, 'r')
        
        self.content_trees = [lxml.etree.parse(StringIO(self.infile.read(filename)))
                              for filename in self.templated_files]
        self.tree_roots = [tree.getroot() for tree in self.content_trees]
        
#        self.py3ocontent = lxml.etree.parse(StringIO(self.infile.read("content.xml")))
#        self.py3oroot = self.py3ocontent.getroot()
        self.__prepare_namespaces()

    def __prepare_namespaces(self):
        """create proper namespaces for our document
        """
        # create needed namespaces
        self.namespaces = dict(
            text="urn:text",
            draw="urn:draw",
            table="urn:table",
            office="urn:office",
            xlink="urn:xlink",
            svg="urn:svg",
            )

        # copy namespaces from original docs
        for tree_root in self.tree_roots:
            self.namespaces.update(tree_root.nsmap)

        # remove any "root" namespace as lxml.xpath do not support them
        self.namespaces.pop(None, None)

        # declare the genshi namespace
        self.namespaces['py'] = GENSHI_URI
        # declare our own namespace
        self.namespaces['py3o'] = PY3O_URI

    def __handle_instructions(self):
        # find all links that have a py3o
        xpath_expr = "//text:a[starts-with(@xlink:href, 'py3o://')]"

        opened_starts = list()
        starting_tags = list()
        closing_tags = dict()
        
        for content_tree in self.content_trees:
            for link in content_tree.xpath(xpath_expr, namespaces=self.namespaces):
                py3o_statement = urllib.unquote(link.attrib['{%s}href' % self.namespaces['xlink']])
                # remove the py3o://
                py3o_base = py3o_statement[7:]
    
                if not py3o_base.startswith("/"):
                    opened_starts.append((content_tree, link))
                    starting_tags.append((content_tree, link, py3o_base))
    
                else:
                    closing_tags[id(opened_starts.pop()[1])] = (content_tree, link)

        return starting_tags, closing_tags

    def __handle_link(self, content_tree, link, py3o_base, closing_link):
        """transform a py3o link into a proper Genshi statement
        rebase a py3o link a a proper place in the tree
        to be ready for Genshi replacement
        """
        if not link[0].text == py3o_base:
            msg = "url and text do not match in '%s'" % link.text
            raise ValueError(msg)

        if link.getparent().getparent().tag == "{%s}table-cell" % self.namespaces['table']:
            # we are in a table
            opening_paragraph = link.getparent()
            opening_cell = opening_paragraph.getparent()
            opening_row = opening_cell.getparent()

            # same for closing
            closing_paragraph = closing_link.getparent()
            closing_cell = closing_paragraph.getparent()
            closing_row = closing_cell.getparent()

        elif link.getparent().tag == "{%s}p" % self.namespaces['text']:
            # we are in a text paragraph
            opening_row = link.getparent()
            closing_row = closing_link.getparent()

        else:
            raise NotImplementedError("We handle urls in tables or text paragraph only")

        # max split is one
        instruction, instruction_value = py3o_base.split("=", 1)
        instruction_value = instruction_value.strip('"')

        attribs = dict()
        attribs['{%s}strip' % GENSHI_URI] = 'True'
        attribs['{%s}%s' % (GENSHI_URI, instruction)] = instruction_value

        genshi_node = lxml.etree.Element('span',
{'fraction_non_alphanumeric': 0.09484990889038074, 'fraction_numerical': 0.005082957705955692, 'mean_word_length': 2.2244897959183674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1032879', 'n_tokens_mistral': 3045, 'n_tokens_neox': 2951, 'n_words': 815}