<commit_before>#!/usr/bin/env python
"""
The XML parsing module.

Parses odML files. Can be invoked standalone:

    python -m odml.tools.xmlparser file.odml
"""
#TODO make this module a parser class, allow arguments (e.g. skip_errors=1 to parse even broken documents)
import sys

from odml import format
from lxml import etree as ET
from lxml.builder import E

# this is needed for py2exe to include lxml completely
from lxml import _elementpath as _dummy

try:
    unicode = unicode
except NameError:
    unicode = str

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

format.Document._xml_name = "odML"
format.Section._xml_name = "section"
format.Property._xml_name = "property"
format.Value._xml_name = "value"

format.Document._xml_attributes = {}
format.Section._xml_attributes = {'name': None}  # attribute 'name' maps to 'name', but writing it as a tag is preferred
format.Property._xml_attributes = {}
format.Value._xml_attributes = {}
format.Value._xml_content = 'value'

XML_VERSION = "1"


class XMLWriter:
    """
    Creates XML nodes storing the information of an odML Document
    """
    header = """<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet  type="text/xsl" href="odmlTerms.xsl"?>
<?xml-stylesheet  type="text/xsl" href="odml.xsl"?>
"""

    def __init__(self, odml_document):
        self.doc = odml_document

    @staticmethod
    def save_element(e):
        """
        returns an xml node for the odML object e
        """
        fmt = e._format
        if hasattr(fmt, "_xml_content"):
            val = getattr(e, fmt.map(fmt._xml_content))
            if val is None:
                val = ''
            cur = E(fmt._name, val)
        else:
            cur = E(fmt._name)

        # generate attributes
        if isinstance(fmt, format.Document.__class__):
            cur.attrib['version'] = XML_VERSION

        for k, v in fmt._xml_attributes.items():
            if not v or not hasattr(e, fmt.map(v)):
                continue

            val = getattr(e, fmt.map(v))
            if val is None:
                continue # no need to save this
            if sys.version_info < (3, 0):
                cur.attrib[k] = unicode(val)
            else:
                cur.attrib[k] = str(val)

        # generate elements
        for k in fmt._args:
            if (k in fmt._xml_attributes and fmt._xml_attributes[k] is not None) or not hasattr(e, fmt.map(k)) \
                    or (hasattr(fmt, "_xml_content") and fmt._xml_content == k):
                continue

            val = getattr(e, fmt.map(k))
            if val is None:
                continue

            if isinstance(val, list):
                for v in val:
                    if v is None:
                        continue
                    ele = XMLWriter.save_element(v)
                    cur.append(ele)
            else:
                if sys.version_info < (3,):
                    ele = E(k, unicode(val))
                else:
                    ele = E(k, str(val))
                # ele = E(k, unicode(val))
                cur.append(ele)
        return cur

    def __str__(self):
        return ET.tounicode(self.save_element(self.doc), pretty_print=True)

    def __unicode__(self):
        return ET.tounicode(self.save_element(self.doc), pretty_print=True)

    def write_file(self, filename):
        # calculate the data before opening the file in case we get any
        # exception
        if sys.version_info < (3, ):
            data = unicode(self).encode('utf-8')
        else:
            data = str(self)

        f = open(filename, "w")
        f.write(self.header)
        f.write(data)
        f.close()


def load(filename):
    """
    shortcut function for XMLReader().fromFile(open(filename))
    """
    return XMLReader().fromFile(open(filename))


class ParserException(Exception):
    pass


class XMLReader(object):
    """
    A reader to parse xml-files or strings into odml data structures

    Usage:

        >>> doc = XMLReader().fromFile(open("file.odml"))
    """
    def __init__(self, ignore_errors=False, filename=None):
        self.parser = ET.XMLParser(remove_comments=True)
        self.tags = dict([(obj._xml_name, obj) for obj in format.__all__])
        self.ignore_errors = ignore_errors
        self.filename = filename

    def fromFile(self, xml_file):
        """
        parse the datastream from a file like object *xml_file*
        and return an odml data structure
        """
        try:
            root = ET.parse(xml_file, self.parser).getroot()
        except ET.XMLSyntaxError as e:
            raise ParserException(e.message)
        return self.parse_element(root)

    def fromString(self, string):
        try:
            root = ET.XML(string, self.parser)
        except ET.XMLSyntaxError as e:
            raise ParserException(e.message)
        return self.parse_element(root)

    def check_mandatory_arguments(self, data, ArgClass, tag_name, node):
        for k, v in ArgClass._args.items():
            if v != 0 and not ArgClass.map(k) in data:
                self.error("missing element <%s> within <%s> tag" % (k, tag_name) + repr(data), node)

    def is_valid_argument(self, tag_name, ArgClass, parent_node, child=None):
        if tag_name not in ArgClass._args:
            self.error("Invalid element <%s> inside <%s> tag" % (tag_name, parent_node.tag), parent_node if child is None else child)

    def error(self, msg, elem):
        if elem is not None:
            msg += " (line %d)" % elem.sourceline
        if self.ignore_errors:
            return self.warn(msg, elem)
        raise ParserException(msg)

    def warn(self, msg, elem):<commit_msg>replace e.message by e.msg<commit_after>#!/usr/bin/env python
"""
The XML parsing module.

Parses odML files. Can be invoked standalone:

    python -m odml.tools.xmlparser file.odml
"""
#TODO make this module a parser class, allow arguments (e.g. skip_errors=1 to parse even broken documents)
import sys

from odml import format
from lxml import etree as ET
from lxml.builder import E

# this is needed for py2exe to include lxml completely
from lxml import _elementpath as _dummy

try:
    unicode = unicode
except NameError:
    unicode = str

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

format.Document._xml_name = "odML"
format.Section._xml_name = "section"
format.Property._xml_name = "property"
format.Value._xml_name = "value"

format.Document._xml_attributes = {}
format.Section._xml_attributes = {'name': None}  # attribute 'name' maps to 'name', but writing it as a tag is preferred
format.Property._xml_attributes = {}
format.Value._xml_attributes = {}
format.Value._xml_content = 'value'

XML_VERSION = "1"


class XMLWriter:
    """
    Creates XML nodes storing the information of an odML Document
    """
    header = """<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet  type="text/xsl" href="odmlTerms.xsl"?>
<?xml-stylesheet  type="text/xsl" href="odml.xsl"?>
"""

    def __init__(self, odml_document):
        self.doc = odml_document

    @staticmethod
    def save_element(e):
        """
        returns an xml node for the odML object e
        """
        fmt = e._format
        if hasattr(fmt, "_xml_content"):
            val = getattr(e, fmt.map(fmt._xml_content))
            if val is None:
                val = ''
            cur = E(fmt._name, val)
        else:
            cur = E(fmt._name)

        # generate attributes
        if isinstance(fmt, format.Document.__class__):
            cur.attrib['version'] = XML_VERSION

        for k, v in fmt._xml_attributes.items():
            if not v or not hasattr(e, fmt.map(v)):
                continue

            val = getattr(e, fmt.map(v))
            if val is None:
                continue # no need to save this
            if sys.version_info < (3, 0):
                cur.attrib[k] = unicode(val)
            else:
                cur.attrib[k] = str(val)

        # generate elements
        for k in fmt._args:
            if (k in fmt._xml_attributes and fmt._xml_attributes[k] is not None) or not hasattr(e, fmt.map(k)) \
                    or (hasattr(fmt, "_xml_content") and fmt._xml_content == k):
                continue

            val = getattr(e, fmt.map(k))
            if val is None:
                continue

            if isinstance(val, list):
                for v in val:
                    if v is None:
                        continue
                    ele = XMLWriter.save_element(v)
                    cur.append(ele)
            else:
                if sys.version_info < (3,):
                    ele = E(k, unicode(val))
                else:
                    ele = E(k, str(val))
                # ele = E(k, unicode(val))
                cur.append(ele)
        return cur

    def __str__(self):
        return ET.tounicode(self.save_element(self.doc), pretty_print=True)

    def __unicode__(self):
        return ET.tounicode(self.save_element(self.doc), pretty_print=True)

    def write_file(self, filename):
        # calculate the data before opening the file in case we get any
        # exception
        if sys.version_info < (3, ):
            data = unicode(self).encode('utf-8')
        else:
            data = str(self)

        f = open(filename, "w")
        f.write(self.header)
        f.write(data)
        f.close()


def load(filename):
    """
    shortcut function for XMLReader().fromFile(open(filename))
    """
    return XMLReader().fromFile(open(filename))


class ParserException(Exception):
    pass


class XMLReader(object):
    """
    A reader to parse xml-files or strings into odml data structures

    Usage:

        >>> doc = XMLReader().fromFile(open("file.odml"))
    """
    def __init__(self, ignore_errors=False, filename=None):
        self.parser = ET.XMLParser(remove_comments=True)
        self.tags = dict([(obj._xml_name, obj) for obj in format.__all__])
        self.ignore_errors = ignore_errors
        self.filename = filename

    def fromFile(self, xml_file):
        """
        parse the datastream from a file like object *xml_file*
        and return an odml data structure
        """
        try:
            root = ET.parse(xml_file, self.parser).getroot()
        except ET.XMLSyntaxError as e:
            raise ParserException(e.msg)
        return self.parse_element(root)

    def fromString(self, string):
        try:
            root = ET.XML(string, self.parser)
        except ET.XMLSyntaxError as e:
            raise ParserException(e.msg)
        return self.parse_element(root)

    def check_mandatory_arguments(self, data, ArgClass, tag_name, node):
        for k, v in ArgClass._args.items():
            if v != 0 and not ArgClass.map(k) in data:
                self.error("missing element <%s> within <%s> tag" % (k, tag_name) + repr(data), node)

    def is_valid_argument(self, tag_name, ArgClass, parent_node, child=None):
        if tag_name not in ArgClass._args:
            self.error("Invalid element <%s> inside <%s> tag" % (tag_name, parent_node.tag), parent_node if child is None else child)

    def error(self, msg, elem):
        if elem is not None:
            msg += " (line %d)" % elem.sourceline
        if self.ignore_errors:
            return self.warn(msg, elem)
        raise ParserException(msg)

    def warn(self, msg, elem):
{'fraction_non_alphanumeric': 0.10657814274482397, 'fraction_numerical': 0.0020966192015375207, 'mean_word_length': 2.115949918345128, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 2, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 60}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1271889', 'n_tokens_mistral': 3462, 'n_tokens_neox': 3327, 'n_words': 976}