<commit_before>import xml.etree.ElementTree as ET
from warnings import warn
from pydmrs.components import RealPred, GPred
from pydmrs.core import Link, ListDmrs
from pydmrs._exceptions import *


def loads_xml(bytestring, encoding=None, cls=ListDmrs, **kwargs):
    """
    Currently processes "<dmrs>...</dmrs>"
    To be updated for "<dmrslist>...</dmrslist>"...
    Expects a bytestring; to load from a string instead, specify encoding
    Produces a ListDmrs by default; for a different type, specify cls
    """
    if encoding:
        bytestring = bytestring.encode(encoding)
    xml = ET.XML(bytestring)

    dmrs = cls(**kwargs)

    dmrs.cfrom = int(xml.get('cfrom')) if 'cfrom' in xml.attrib else None
    dmrs.cto = int(xml.get('cto')) if 'cto' in xml.attrib else None
    dmrs.surface = xml.get('surface')
    dmrs.ident = int(xml.get('ident')) if 'ident' in xml.attrib else None
    index_id = int(xml.get('index')) if 'index' in xml.attrib else None
    top_id = None

    for elem in xml:
        if elem.tag == 'node':
            nodeid = int(elem.get('nodeid')) if 'nodeid' in elem.attrib else None
            cfrom = int(elem.get('cfrom')) if 'cfrom' in elem.attrib else None
            cto = int(elem.get('cto')) if 'cto' in elem.attrib else None
            surface = elem.get('surface')
            base = elem.get('base')
            carg = elem.get('carg')

            pred = None
            sortinfo = None
            for sub in elem:
                if sub.tag == 'realpred':
                    try:
                        pred = RealPred(sub.get('lemma'), sub.get('pos'), sub.get('sense'))
                    except PydmrsValueError:
                        # If the whole pred name is under 'lemma', rather than split between 'lemma', 'pos', 'sense'
                        pred = RealPred.from_string(sub.get('lemma'))
                        warn("RealPred given as string rather than lemma, pos, sense", PydmrsWarning)
                elif sub.tag == 'gpred':
                    try:
                        pred = GPred.from_string(sub.text)
                    except PydmrsValueError:
                        # If the string is actually for a RealPred, not a GPred
                        pred = RealPred.from_string(sub.text)
                        warn("RealPred string found in a <gpred> tag", PydmrsWarning)
                elif sub.tag == 'sortinfo':
                    sortinfo = sub.attrib
                else:
                    raise PydmrsValueError(sub.tag)

            dmrs.add_node(cls.Node(nodeid=nodeid, pred=pred, carg=carg, sortinfo=sortinfo, cfrom=cfrom, cto=cto, surface=surface, base=base))

        elif elem.tag == 'link':
            start = int(elem.get('from'))
            end = int(elem.get('to'))

            if start == 0:
                top_id = end
            else:
                rargname = None
                post = None
                for sub in elem:
                    if sub.tag == 'rargname':
                        rargname = sub.text
                    elif sub.tag == 'post':
                        post = sub.text
                    else:
                        raise PydmrsValueError(sub.tag)
                dmrs.add_link(Link(start, end, rargname, post))
        else:
            raise PydmrsValueError(elem.tag)

    if top_id:
        dmrs.top = dmrs[top_id]
    if index_id:
        dmrs.index = dmrs[index_id]

    return dmrs


def load_xml(filehandle, cls=ListDmrs):
    """
    Load a DMRS from a file
    NB: read file as bytes!
    Produces a ListDmrs by default; for a different type, specify cls
    """
    return cls.loads(filehandle.read(), cls=cls)


def dumps_xml(dmrs, encoding=None):
    """
    Currently creates "<dmrs>...</dmrs>"
    To be updated for "<dmrslist>...</dmrslist>"...
    Returns a bytestring; to return a string instead, specify encoding
    """
    xdmrs = ET.Element('dmrs')
    if dmrs.index is not None:
        xdmrs.set('index', str(dmrs.index.nodeid))
    if dmrs.cfrom is not None and dmrs.cto is not None:
        xdmrs.set('cfrom', str(dmrs.cfrom))
        xdmrs.set('cto', str(dmrs.cto))
    for nodeid in sorted(dmrs):
        xnode = ET.SubElement(xdmrs, 'node')
        xnode.set('nodeid', str(nodeid))
        node = dmrs[nodeid]
        if node.cfrom is not None and node.cto is not None:
            xnode.set('cfrom', str(node.cfrom))
            xnode.set('cto', str(node.cto))
        if node.carg:
            xnode.set('carg', '"{}"'.format(node.carg))
        if isinstance(node.pred, GPred):
            xpred = ET.SubElement(xnode, 'gpred')
            xpred.text = str(node.pred) + '_rel'
        elif isinstance(node.pred, RealPred):
            xpred = ET.SubElement(xnode, 'realpred')
            xpred.set('lemma', node.pred.lemma)
            xpred.set('pos', node.pred.pos)
            if node.pred.sense:
                xpred.set('sense', node.pred.sense)<commit_msg>Remove extra quotes when serializing carg.<commit_after>import xml.etree.ElementTree as ET
from warnings import warn
from pydmrs.components import RealPred, GPred
from pydmrs.core import Link, ListDmrs
from pydmrs._exceptions import *


def loads_xml(bytestring, encoding=None, cls=ListDmrs, **kwargs):
    """
    Currently processes "<dmrs>...</dmrs>"
    To be updated for "<dmrslist>...</dmrslist>"...
    Expects a bytestring; to load from a string instead, specify encoding
    Produces a ListDmrs by default; for a different type, specify cls
    """
    if encoding:
        bytestring = bytestring.encode(encoding)
    xml = ET.XML(bytestring)

    dmrs = cls(**kwargs)

    dmrs.cfrom = int(xml.get('cfrom')) if 'cfrom' in xml.attrib else None
    dmrs.cto = int(xml.get('cto')) if 'cto' in xml.attrib else None
    dmrs.surface = xml.get('surface')
    dmrs.ident = int(xml.get('ident')) if 'ident' in xml.attrib else None
    index_id = int(xml.get('index')) if 'index' in xml.attrib else None
    top_id = None

    for elem in xml:
        if elem.tag == 'node':
            nodeid = int(elem.get('nodeid')) if 'nodeid' in elem.attrib else None
            cfrom = int(elem.get('cfrom')) if 'cfrom' in elem.attrib else None
            cto = int(elem.get('cto')) if 'cto' in elem.attrib else None
            surface = elem.get('surface')
            base = elem.get('base')
            carg = elem.get('carg')

            pred = None
            sortinfo = None
            for sub in elem:
                if sub.tag == 'realpred':
                    try:
                        pred = RealPred(sub.get('lemma'), sub.get('pos'), sub.get('sense'))
                    except PydmrsValueError:
                        # If the whole pred name is under 'lemma', rather than split between 'lemma', 'pos', 'sense'
                        pred = RealPred.from_string(sub.get('lemma'))
                        warn("RealPred given as string rather than lemma, pos, sense", PydmrsWarning)
                elif sub.tag == 'gpred':
                    try:
                        pred = GPred.from_string(sub.text)
                    except PydmrsValueError:
                        # If the string is actually for a RealPred, not a GPred
                        pred = RealPred.from_string(sub.text)
                        warn("RealPred string found in a <gpred> tag", PydmrsWarning)
                elif sub.tag == 'sortinfo':
                    sortinfo = sub.attrib
                else:
                    raise PydmrsValueError(sub.tag)

            dmrs.add_node(cls.Node(nodeid=nodeid, pred=pred, carg=carg, sortinfo=sortinfo, cfrom=cfrom, cto=cto, surface=surface, base=base))

        elif elem.tag == 'link':
            start = int(elem.get('from'))
            end = int(elem.get('to'))

            if start == 0:
                top_id = end
            else:
                rargname = None
                post = None
                for sub in elem:
                    if sub.tag == 'rargname':
                        rargname = sub.text
                    elif sub.tag == 'post':
                        post = sub.text
                    else:
                        raise PydmrsValueError(sub.tag)
                dmrs.add_link(Link(start, end, rargname, post))
        else:
            raise PydmrsValueError(elem.tag)

    if top_id:
        dmrs.top = dmrs[top_id]
    if index_id:
        dmrs.index = dmrs[index_id]

    return dmrs


def load_xml(filehandle, cls=ListDmrs):
    """
    Load a DMRS from a file
    NB: read file as bytes!
    Produces a ListDmrs by default; for a different type, specify cls
    """
    return cls.loads(filehandle.read(), cls=cls)


def dumps_xml(dmrs, encoding=None):
    """
    Currently creates "<dmrs>...</dmrs>"
    To be updated for "<dmrslist>...</dmrslist>"...
    Returns a bytestring; to return a string instead, specify encoding
    """
    xdmrs = ET.Element('dmrs')
    if dmrs.index is not None:
        xdmrs.set('index', str(dmrs.index.nodeid))
    if dmrs.cfrom is not None and dmrs.cto is not None:
        xdmrs.set('cfrom', str(dmrs.cfrom))
        xdmrs.set('cto', str(dmrs.cto))
    for nodeid in sorted(dmrs):
        xnode = ET.SubElement(xdmrs, 'node')
        xnode.set('nodeid', str(nodeid))
        node = dmrs[nodeid]
        if node.cfrom is not None and node.cto is not None:
            xnode.set('cfrom', str(node.cfrom))
            xnode.set('cto', str(node.cto))
        if node.carg:
            xnode.set('carg', '{}'.format(node.carg))
        if isinstance(node.pred, GPred):
            xpred = ET.SubElement(xnode, 'gpred')
            xpred.text = str(node.pred) + '_rel'
        elif isinstance(node.pred, RealPred):
            xpred = ET.SubElement(xnode, 'realpred')
            xpred.set('lemma', node.pred.lemma)
            xpred.set('pos', node.pred.pos)
            if node.pred.sense:
                xpred.set('sense', node.pred.sense)
{'fraction_non_alphanumeric': 0.11868508621558939, 'fraction_numerical': 0.00020167389331451045, 'mean_word_length': 1.8296718972895862, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1825752', 'n_tokens_mistral': 3139, 'n_tokens_neox': 3011, 'n_words': 808}