<commit_before>from datetime import datetime
from lxml import etree
from lxml.builder import ElementMaker
from eulcore.xpath import ast, parse, serialize

__all__ = [
    'StringField', 'StringListField',
    'IntegerField', 'IntegerListField',
    'NodeField', 'NodeListField',
    'ItemField', 'SimpleBooleanField'
# NOTE: DateField and DateListField are undertested and underdocumented. If
#   you really need them, you should import them explicitly. Or even better,
#   flesh them out so they can be properly released.
]

# base class for all fields

class Field(object):

    # track each time a Field instance is created, to retain order
    creation_counter = 0

    def __init__(self, xpath, manager, mapper):
        # compile xpath in order to catch an invalid xpath at load time
        etree.XPath(xpath)
        # NOTE: not saving compiled xpath because namespaces must be
        # passed in at compile time when evaluating an etree.XPath on a node
        self.xpath = xpath
        self.manager = manager
        self.mapper = mapper

        # adjust creation counter, save local copy of current count
        self.creation_counter = Field.creation_counter
        Field.creation_counter += 1

        # determine parent xpath, node name, node type 
        effective_xpath = parse(xpath)
        self.node_info = {}
        
        if isinstance(effective_xpath, ast.BinaryExpression) and \
                effective_xpath.op == '/':
            self.node_info['parent_xpath'] = serialize(effective_xpath.left)
            effective_xpath = effective_xpath.right
            
        if isinstance(effective_xpath, ast.Step) and \
                isinstance(effective_xpath.node_test, ast.NameTest):
            self.node_info['type'] = effective_xpath.axis or 'child'
            if self.node_info['type'] == '@':
                self.node_info['type'] = 'attribute'
            self.node_info['prefix'] = effective_xpath.node_test.prefix
            self.node_info['name'] = effective_xpath.node_test.name

    def get_for_node(self, node, context):
        return self.manager.get(self.xpath, node, context, self.mapper.to_python)

    def set_for_node(self, node, context, value):
        return self.manager.set(self.xpath, node, context, self.mapper.to_xml, value, self.node_info)


# data mappers to translate between identified xml nodes and Python values

class Mapper(object):
    # generic mapper to_xml function
    def to_xml(self, value):
        return unicode(value)

class StringMapper(Mapper):
    XPATH = etree.XPath('string()')
    def __init__(self, normalize=False):
        if normalize:
            self.XPATH = etree.XPath('normalize-space(string())')
        
    def to_python(self, node):
        if isinstance(node, basestring):
            return node
        return self.XPATH(node)
       
class NumberMapper(Mapper):
    XPATH = etree.XPath('number()')
    def to_python(self, node):
        if isinstance(node, basestring):
            return int(node)     # FIXME: not equivalent to xpath number()...
        return self.XPATH(node)

class SimpleBooleanMapper(Mapper):
    XPATH = etree.XPath('string()')
    def __init__(self, true, false):
        self.true = true
        self.false = false
        
    def to_python(self, node):
        if isinstance(node, basestring):
            value = node
        else:
            value = self.XPATH(node)
        if value == str(self.true):
            return True
        if value == str(self.false):
            return False        
        # what happens if it is neither of these?
        raise Exception("Boolean field value '%s' is neither '%s' nor '%s'" % (value, self.true, self.false))
        

    def to_xml(self, value):
        if value:
            return str(self.true)
        else:
            return str(self.false)

class DateMapper(object):
    XPATH = etree.XPath('string()')
    def to_python(self, node):
        if isinstance(node, basestring):
            rep = node
        else:
            rep = self.XPATH(node)
        if rep.endswith('Z'): # strip Z
            rep = rep[:-1]
        if rep[-6] in '+-': # strip tz
            rep = rep[:-6]
        try:
            dt = datetime.strptime(rep, '%Y-%m-%dT%H:%M:%S')
        except ValueError, v:
            # if initial format fails, attempt to parse with microseconds
            dt = datetime.strptime(rep, '%Y-%m-%dT%H:%M:%S.%f')
        return dt

    def to_xml(self, dt):
        # NOTE: untested!  this is probably close to what we need, but should be tested
        return unicode(dt.isoformat())

class NullMapper(object):
    def to_python(self, node):
        return node

class NodeMapper(object):
    def __init__(self, node_class):
        self.node_class = node_class

    def to_python(self, node):
        return self.node_class(node)

# managers to map operations to either a single identified node or a
# list of them

class SingleNodeManager(object):
    def get(self, xpath, node, context, to_python):
        match = self.find_xml_node(xpath, node, context)
        if match is not None:
            return to_python(match)

    def set(self, xpath, node, context, to_xml, value, node_info):
        match = self.find_xml_node(xpath, node, context)
        if match == None:
            match = self.create_xml_node(xpath, node, context, node_info)
            # create_xml_node() throws an exception on failure

        return self.set_in_xml(match, to_xml(value), context, node_info)

    def find_xml_node(self, xpath, node, context):
        matches = node.xpath(xpath, **context)
        if matches:
            return matches[0]

    def create_xml_node(self, xpath, node, context, node_info):
        # a node can be created if:

        # relative path and the parent node exists
        if 'parent_xpath' in node_info:
            parent_nodeset = node.xpath(node_info['parent_xpath'], **context)
            if len(parent_nodeset) != 1:
                msg = ("Missing element for '%s', and node creation is " + \
                       "supported only when parent xpath '%s' evaluates " + \
                       "to a single node. Instead, it evaluates to %d.") % \
                       (repr(xpath), repr(parent_xpath), len(parent_nodeset))
                raise Exception(msg)
            # otherwise, we found the parent.
            parent_node = parent_nodeset[0]
        else:
            # if there is no parent xpath, create missing node under the current node
            parent_node = node

        # and if the last part of the path is a simple, unpredicated child
        # or attribute
        if 'type' in node_info:            
            if node_info['type'] == 'child':
                return self.create_child_node(parent_node, context, node_info)
            elif node_info['type'] == 'attribute':
                return self.create_attribute_node(parent_node, context, node_info)

        # anything else, throw an exception:
        msg = ("Missing element for '%s', and node creation is supported " + \
               "only for simple child and attribute nodes.") % (repr(xpath),)
        raise Exception(msg)

    def create_child_node(self, node, context, node_info):
        opts = {}
        ns_uri = None
        if 'namespaces' in context:
            opts['nsmap'] = context['namespaces']
            if node_info['prefix'] is not None:
                ns_uri = context['namespaces'][node_info['prefix']]
        E = ElementMaker(namespace=ns_uri, **opts)
        new_node = E(node_info['name'])
        node.append(new_node)<commit_msg>error generation itself should not throw unexpected exceptions.<commit_after>from datetime import datetime
from lxml import etree
from lxml.builder import ElementMaker
from eulcore.xpath import ast, parse, serialize

__all__ = [
    'StringField', 'StringListField',
    'IntegerField', 'IntegerListField',
    'NodeField', 'NodeListField',
    'ItemField', 'SimpleBooleanField'
# NOTE: DateField and DateListField are undertested and underdocumented. If
#   you really need them, you should import them explicitly. Or even better,
#   flesh them out so they can be properly released.
]

# base class for all fields

class Field(object):

    # track each time a Field instance is created, to retain order
    creation_counter = 0

    def __init__(self, xpath, manager, mapper):
        # compile xpath in order to catch an invalid xpath at load time
        etree.XPath(xpath)
        # NOTE: not saving compiled xpath because namespaces must be
        # passed in at compile time when evaluating an etree.XPath on a node
        self.xpath = xpath
        self.manager = manager
        self.mapper = mapper

        # adjust creation counter, save local copy of current count
        self.creation_counter = Field.creation_counter
        Field.creation_counter += 1

        # determine parent xpath, node name, node type 
        effective_xpath = parse(xpath)
        self.node_info = {}
        
        if isinstance(effective_xpath, ast.BinaryExpression) and \
                effective_xpath.op == '/':
            self.node_info['parent_xpath'] = serialize(effective_xpath.left)
            effective_xpath = effective_xpath.right
            
        if isinstance(effective_xpath, ast.Step) and \
                isinstance(effective_xpath.node_test, ast.NameTest):
            self.node_info['type'] = effective_xpath.axis or 'child'
            if self.node_info['type'] == '@':
                self.node_info['type'] = 'attribute'
            self.node_info['prefix'] = effective_xpath.node_test.prefix
            self.node_info['name'] = effective_xpath.node_test.name

    def get_for_node(self, node, context):
        return self.manager.get(self.xpath, node, context, self.mapper.to_python)

    def set_for_node(self, node, context, value):
        return self.manager.set(self.xpath, node, context, self.mapper.to_xml, value, self.node_info)


# data mappers to translate between identified xml nodes and Python values

class Mapper(object):
    # generic mapper to_xml function
    def to_xml(self, value):
        return unicode(value)

class StringMapper(Mapper):
    XPATH = etree.XPath('string()')
    def __init__(self, normalize=False):
        if normalize:
            self.XPATH = etree.XPath('normalize-space(string())')
        
    def to_python(self, node):
        if isinstance(node, basestring):
            return node
        return self.XPATH(node)
       
class NumberMapper(Mapper):
    XPATH = etree.XPath('number()')
    def to_python(self, node):
        if isinstance(node, basestring):
            return int(node)     # FIXME: not equivalent to xpath number()...
        return self.XPATH(node)

class SimpleBooleanMapper(Mapper):
    XPATH = etree.XPath('string()')
    def __init__(self, true, false):
        self.true = true
        self.false = false
        
    def to_python(self, node):
        if isinstance(node, basestring):
            value = node
        else:
            value = self.XPATH(node)
        if value == str(self.true):
            return True
        if value == str(self.false):
            return False        
        # what happens if it is neither of these?
        raise Exception("Boolean field value '%s' is neither '%s' nor '%s'" % (value, self.true, self.false))
        

    def to_xml(self, value):
        if value:
            return str(self.true)
        else:
            return str(self.false)

class DateMapper(object):
    XPATH = etree.XPath('string()')
    def to_python(self, node):
        if isinstance(node, basestring):
            rep = node
        else:
            rep = self.XPATH(node)
        if rep.endswith('Z'): # strip Z
            rep = rep[:-1]
        if rep[-6] in '+-': # strip tz
            rep = rep[:-6]
        try:
            dt = datetime.strptime(rep, '%Y-%m-%dT%H:%M:%S')
        except ValueError, v:
            # if initial format fails, attempt to parse with microseconds
            dt = datetime.strptime(rep, '%Y-%m-%dT%H:%M:%S.%f')
        return dt

    def to_xml(self, dt):
        # NOTE: untested!  this is probably close to what we need, but should be tested
        return unicode(dt.isoformat())

class NullMapper(object):
    def to_python(self, node):
        return node

class NodeMapper(object):
    def __init__(self, node_class):
        self.node_class = node_class

    def to_python(self, node):
        return self.node_class(node)

# managers to map operations to either a single identified node or a
# list of them

class SingleNodeManager(object):
    def get(self, xpath, node, context, to_python):
        match = self.find_xml_node(xpath, node, context)
        if match is not None:
            return to_python(match)

    def set(self, xpath, node, context, to_xml, value, node_info):
        match = self.find_xml_node(xpath, node, context)
        if match == None:
            match = self.create_xml_node(xpath, node, context, node_info)
            # create_xml_node() throws an exception on failure

        return self.set_in_xml(match, to_xml(value), context, node_info)

    def find_xml_node(self, xpath, node, context):
        matches = node.xpath(xpath, **context)
        if matches:
            return matches[0]

    def create_xml_node(self, xpath, node, context, node_info):
        # a node can be created if:

        # relative path and the parent node exists
        if 'parent_xpath' in node_info:
            parent_nodeset = node.xpath(node_info['parent_xpath'], **context)
            if len(parent_nodeset) != 1:
                msg = ("Missing element for '%s', and node creation is " + \
                       "supported only when parent xpath '%s' evaluates " + \
                       "to a single node. Instead, it evaluates to %d.") % \
                       (repr(xpath), repr(node_info['parent_xpath']), \
                        len(parent_nodeset))
                raise Exception(msg)
            # otherwise, we found the parent.
            parent_node = parent_nodeset[0]
        else:
            # if there is no parent xpath, create missing node under the current node
            parent_node = node

        # and if the last part of the path is a simple, unpredicated child
        # or attribute
        if 'type' in node_info:            
            if node_info['type'] == 'child':
                return self.create_child_node(parent_node, context, node_info)
            elif node_info['type'] == 'attribute':
                return self.create_attribute_node(parent_node, context, node_info)

        # anything else, throw an exception:
        msg = ("Missing element for '%s', and node creation is supported " + \
               "only for simple child and attribute nodes.") % (repr(xpath),)
        raise Exception(msg)

    def create_child_node(self, node, context, node_info):
        opts = {}
        ns_uri = None
        if 'namespaces' in context:
            opts['nsmap'] = context['namespaces']
            if node_info['prefix'] is not None:
                ns_uri = context['namespaces'][node_info['prefix']]
        E = ElementMaker(namespace=ns_uri, **opts)
        new_node = E(node_info['name'])
        node.append(new_node)
{'fraction_non_alphanumeric': 0.0990501146413364, 'fraction_numerical': 0.001048149361283983, 'mean_word_length': 2.3179743534014343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 34}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2334599', 'n_tokens_mistral': 4391, 'n_tokens_neox': 4302, 'n_words': 1308}