<commit_before># file test_xmlmap/test_fields.py
# 
#   Copyright 2011 Emory University Libraries
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

#!/usr/bin/env python

from datetime import datetime
import tempfile
import unittest

import eulxml.xmlmap.core as xmlmap
from testcore import main

class TestFields(unittest.TestCase):
    FIXTURE_TEXT = '''
        <foo id='a' xmlns:ex='http://example.com/'>
            <bar>
                <baz>42</baz>
            </bar>
            <bar>
                <baz>13</baz>
            </bar>
            <empty_field/>
            <boolean>
                    <text1>yes</text1>
                    <text2>no</text2>
                    <num1>1</num1>
                    <num2>0</num2>
            </boolean>
            <spacey>           this text
        needs to be
                normalized
            </spacey>
            <date>2010-01-03T02:13:44</date>
            <date>2010-01-03T02:13:44.003</date>
        </foo>
    '''

    namespaces = {'ex' : 'http://example.com/'}

    def setUp(self):
        # parseString wants a url. let's give it a proper one.
        url = '%s#%s.%s' % (__file__, self.__class__.__name__, 'FIXTURE_TEXT')

        self.fixture = xmlmap.parseString(self.FIXTURE_TEXT, url)
        
        self.rel_url = '%s#%s' % (__file__, self.__class__.__name__)

    def _empty_fixture(self):
        return xmlmap.parseString('<root/>', self.rel_url)

    def elf.fixtureestInvalidXpath(self):
        self.assertRaises(Exception, xmlmap.StringField, '["')
        
    def testNodeField(self):
        class TestSubobject(xmlmap.XmlObject):
            ROOT_NAME = 'bar'
            val = xmlmap.StringField('baz')

        class TestObject(xmlmap.XmlObject):
            child = xmlmap.NodeField('bar[1]', TestSubobject, required=True)
            missing = xmlmap.NodeField('missing', TestSubobject)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.child.val, '42')
        self.assertEqual(obj.missing, None)
        # undefined if >1 matched nodes

        # test set
        obj.child = TestSubobject(val='144')
        self.assertEqual(obj.child.val, '144')

        # check required
        self.assertTrue(obj._fields['child'].required)

        # test create_foo functionality
        class CreateTestObject(xmlmap.XmlObject):
            missing = xmlmap.NodeField('missing', TestSubobject)
        obj = CreateTestObject(self.fixture)
        self.assert_(obj.missing is None)
        obj.create_missing()
        self.assert_(isinstance(obj.missing, TestSubobject))

        # test del
        del obj.missing
        self.assertEqual(obj.missing, None)

        # test DEPRECATED instantiate on get hack
        class GetTestObject(xmlmap.XmlObject):
            missing = xmlmap.NodeField('missing', TestSubobject, instantiate_on_get=True)
        obj = GetTestObject(self.fixture)
        self.assert_(isinstance(obj.missing, TestSubobject),
            "non-existent nodefield is created on get when 'instantiate_on_get' flag is set")

    def testNodeListField(self):
        class TestSubobject(xmlmap.XmlObject):
            val = xmlmap.IntegerField('baz')

        class TestObject(xmlmap.XmlObject):
            children = xmlmap.NodeListField('bar', TestSubobject)
            missing = xmlmap.NodeListField('missing', TestSubobject, required=False)

        obj = TestObject(self.fixture)
        child_vals = [ child.val for child in obj.children ]
        self.assertEqual(child_vals, [42, 13])
        self.assertEqual(obj.missing, [])

        # check required
        self.assertFalse(obj._fields['missing'].required)

        # is_empty
        self.assertFalse(obj.children.is_empty())
        del obj.children
        self.assertTrue(obj.children.is_empty())


    def testStringField(self):
        class TestObject(xmlmap.XmlObject):
            val = xmlmap.StringField('bar[1]/baz', required=True)
            empty = xmlmap.StringField('empty_field', required=False)
            missing = xmlmap.StringField('missing')
            missing_ns = xmlmap.StringField('ex:missing')
            missing_att = xmlmap.StringField('@missing')
            missing_att_ns = xmlmap.StringField('@ex:missing')
            sub_missing = xmlmap.StringField('bar[1]/missing')
            multilevel_missing = xmlmap.StringField('missing_parent/missing_child')
            mixed = xmlmap.StringField('bar[1]')
            id = xmlmap.StringField('@id')
            spacey = xmlmap.StringField('spacey')
            normal_spacey = xmlmap.StringField('spacey', normalize=True)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.val, '42')
        self.assertEqual(obj.missing, None)
        # undefined if >1 matched nodes

        # access normalized and non-normalized versions of string field
        self.assertNotEqual("this text needs to be normalized", obj.spacey)
        self.assertEqual("this text needs to be normalized", obj.normal_spacey)

        # set an existing string value
        obj.val = "forty-two"
        # check that new value is set in the node
        self.assertEqual(obj.node.xpath('string(bar[1]/baz)'), "forty-two")
        # check that new value is accessible via descriptor        
        self.assertEqual(obj.val, 'forty-two')

        # set an attribute
        obj.id = 'z'
        # check that new value is set in the node
        self.assertEqual(obj.node.xpath('string(@id)'), "z")
        # check that new value is accessible via descriptor
        self.assertEqual(obj.id, 'z')

        # set value in an empty node
        obj.empty = "full"
        self.assertEqual(obj.node.xpath('string(empty_field)'), "full")
        # check that new value is accessible via descriptor
        self.assertEqual(obj.empty, 'full')

        # set missing fields
        obj.missing = 'not here'
        self.assertEqual(obj.node.xpath('string(missing)'), 'not here')
        # with ns
        obj.missing_ns = 'over there'
        self.assertEqual(obj.node.xpath('string(ex:missing)', namespaces=self.namespaces),
                    'over there')
        # in attrib
        obj.missing_att = 'out to pasture'
        self.assertEqual(obj.node.xpath('string(@missing)'), 'out to pasture')
        # in attrib with ns
        obj.missing_att_ns = "can't find me!"
        self.assertEqual(obj.node.xpath('string(@ex:missing)',
                        namespaces=self.namespaces), "can't find me!")
        # in subelement
        obj.sub_missing = 'pining (for the fjords)'
        self.assertEqual(obj.node.xpath('string(bar/missing)'), 'pining (for the fjords)')

        # in subelement which is itself missing
        obj.multilevel_missing = 'so, so gone'
        self.assertEqual(obj.node.xpath('string(missing_parent/missing_child)'), 'so, so gone')

        # attempting to set a node that contains non-text nodes - error
        self.assertRaises(Exception, obj.__setattr__, "mixed", "whoops")

        # set an existing string value to None
        # - attribute
        obj.id = None
        # check that attribute was removed from the xml
        self.assertEqual(0, obj.node.xpath('count(@id)'))
        self.assertEqual(None, obj.id)
        # - element
        obj.val = None
        # check that node is removed from the xml
        self.assertEqual(0, obj.node.xpath('count(bar[1]/baz)'))
        self.assertEqual(None, obj.val)

        # set to empty string - should store empty string and NOT remove the node
        # - attribute
        obj.id = ''
        # check that attribute was removed from the xml
        self.assertEqual('', obj.node.xpath('string(@id)'))
        self.assertEqual('', obj.id)
        # - element
        obj.val = ''
        # check that node is removed from the xml
        self.assertEqual('', obj.node.xpath('string(bar[1]/baz)'))
        self.assertEqual('', obj.val)

        # delete
        delattr(obj, 'spacey')
        # check that node is removed from the xml
        self.assertEqual(0, obj.node.xpath('count(spacey)'))
        self.assertEqual(None, obj.spacey)

        # check required
        self.assertTrue(obj._fields['val'].required)
        self.assertFalse(obj._fields['empty'].required)
        self.assertEqual(None, obj._fields['missing'].required)

    def testTextNode(self):
        # special case for text()
        class TextObject(xmlmap.XmlObject):
            text_node = xmlmap.StringField('text()')
            nested_text =  xmlmap.StringField('nest[@type="feather"]/text()')
            missing_text = xmlmap.StringField('missing[@type="foo"]/bar/text()')
            m_nested_attr = xmlmap.StringField('missing[@type="foo"]/bar/@label')
            m_nested_text = xmlmap.StringField('missing[@type="foo"]/baz')
            nested_child_pred = xmlmap.StringField('missing[@type="foo"][baz="bah"]/txt')
        # parseString wants a url. let's give it a proper one.
        url = '%s#%s.%s' % (__file__, self.__class__.__name__, 'TEXT_XML_FIXTURES')
        xml = '''<text>some text<nest type='feather'>robin</nest></text>'''
        obj = TextObject(xmlmap.parseString(xml, url))

        self.assertEqual('some text', obj.text_node)
        
        # set text node
        obj.text_node = 'la'
        self.assertEqual('la', obj.node.xpath('string(text())'))
        self.assertEqual('la', obj.text_node)
        # set to empty string
        obj.text_node = ''
        self.assertEqual('', obj.node.xpath('string(text())'))
        self.assertEqual('', obj.text_node)
        # set to value then set to None - text should be cleared out
        obj.text_node = 'la'
        obj.text_node = None
        self.assertEqual('', obj.node.xpath('string(text())'))
        self.assertEqual('', obj.text_node)

        self.assertEqual('robin', obj.nested_text)

        # create parent of a text node
        obj.missing_text = 'tra'
        self.assertEqual('tra', obj.node.xpath('string(missing/bar/text())'))
        self.assertEqual('tra', obj.missing_text)

    def testStringListField(self):
        class TestObject(xmlmap.XmlObject):
            vals = xmlmap.StringListField('bar/baz', required=True)
            missing = xmlmap.StringListField('missing')
            spacey = xmlmap.StringListField('spacey', normalize=True)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.vals, ['42', '13'])
        self.assertEqual(obj.missing, [])

        # access normalized string list field
        self.assertEqual("this text needs to be normalized", obj.spacey[0])

        # delete list field
        delattr(obj, 'spacey')
        # check that node is removed from the xml
        self.assertEqual(0, obj.node.xpath('count(spacey)'))
        self.assertEqual([], obj.spacey)

        # check required
        self.assertTrue(obj._fields['vals'].required)
        

    def testIntegerField(self):
        class TestObject(xmlmap.XmlObject):
            val = xmlmap.IntegerField('bar[2]/baz', required=True)
            count = xmlmap.IntegerField('count(//bar)')
            missing = xmlmap.IntegerField('missing')
            nan = xmlmap.IntegerField('@id')

        obj = TestObject(self.fixture)
        self.assertEqual(obj.val, 13)
        self.assertEqual(obj.count, 2)
        self.assertEqual(obj.missing, None)
        self.assertEqual(obj.nan, None)
        # undefined if >1 matched nodes

        # set an integer value
        obj.val = 14
        # check that new value is set in the node
        self.assertEqual(obj.node.xpath('number(bar[2]/baz)'), 14)
        # check that new value is accessible via descriptor
        self.assertEqual(obj.val, 14)

        # check required
        self.assertTrue(obj._fields['val'].required)

    def testIntegerListField(self):
        class TestObject(xmlmap.XmlObject):
            vals = xmlmap.IntegerListField('bar/baz')
            missing = xmlmap.IntegerListField('missing', required=False)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.vals, [42, 13])
        self.assertEqual(obj.missing, [])

        # check required
        self.assertFalse(obj._fields['missing'].required)

    def testItemField(self):
        class TestObject(xmlmap.XmlObject):
            letter = xmlmap.ItemField('substring(bar/baz, 1, 1)')
            missing = xmlmap.ItemField('missing', required=False)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.letter, '4')
        self.assertEqual(obj.missing, None)

        # check required
        self.assertFalse(obj._fields['missing'].required)

    def testBooleanField(self):        
        class TestObject(xmlmap.XmlObject):
            txt_bool1 = xmlmap.SimpleBooleanField('boolean/text1', 'yes', 'no', required=False)
            txt_bool2 = xmlmap.SimpleBooleanField('boolean/text2', 'yes', 'no')
            num_bool1 = xmlmap.SimpleBooleanField('boolean/num1', 1, 0)
            num_bool2 = xmlmap.SimpleBooleanField('boolean/num2', 1, 0)
            opt_elem_bool = xmlmap.SimpleBooleanField('boolean/opt', 'yes', None)
            opt_attr_bool = xmlmap.SimpleBooleanField('boolean/@opt', 'yes', None)

        #obj = TestObject(self.fixture.documentElement)
        obj = TestObject(self.fixture)
        self.assertEqual(obj.txt_bool1, True)
        self.assertEqual(obj.txt_bool2, False)
        self.assertEqual(obj.num_bool1, True)
        self.assertEqual(obj.num_bool2, False)
        self.assertEqual(obj.opt_elem_bool, False)
        self.assertEqual(obj.opt_attr_bool, False)

        # set text boolean
        obj.txt_bool1 = False
        # check that new value is set in the node
        self.assertEqual(obj.node.xpath('string(boolean/text1)'), 'no')
        # check that new value is accessible via descriptor
        self.assertEqual(obj.txt_bool1, False)

        # set numeric boolean
        obj.num_bool1 = False
        # check for new new value in the node and via descriptor
        self.assertEqual(obj.node.xpath('number(boolean/num1)'), 0)
        self.assertEqual(obj.num_bool1, False)

        # set optional element boolean
        obj.opt_elem_bool = True
        self.assertEqual(obj.node.xpath('string(boolean/opt)'), 'yes')
        self.assertEqual(obj.opt_elem_bool, True)
        obj.opt_elem_bool = False
        self.assertEqual(obj.node.xpath('count(boolean/opt)'), 0)
        self.assertEqual(obj.opt_elem_bool, False)

        # set optional attribute boolean
        obj.opt_attr_bool = True
        self.assertEqual(obj.node.xpath('string(boolean/@opt)'), 'yes')
        self.assertEqual(obj.opt_attr_bool, True)
        obj.opt_attr_bool = False
        self.assertEqual(obj.node.xpath('count(boolean/@opt)'), 0)
        self.assertEqual(obj.opt_attr_bool, False)

        # check required
        self.assertFalse(obj._fields['txt_bool1'].required)


    # FIXME: DateField and DateListField are hacked together. Until we
    #   work up some proper parsing and good testing for them, they should
    #   be considered untested and undocumented features.

    
    def testDateTimeField(self):        
        class TestObject(xmlmap.XmlObject):
            date = xmlmap.DateTimeField('date')
            dates = xmlmap.DateTimeListField('date')

        obj = TestObject(self.fixture)
        # fields should be datetime objects
        self.assert_(isinstance(obj.date, datetime))
        self.assert_(isinstance(obj.dates[1], datetime))
        # inspect date parsing
        self.assertEqual(2010, obj.date.year)
        self.assertEqual(1, obj.date.month)
        self.assertEqual(3, obj.date.day)
        self.assertEqual(2, obj.date.hour)
        self.assertEqual(13, obj.date.minute)
        self.assertEqual(44, obj.date.second)
        # microsecond date parsing
        self.assertEqual(3000, obj.dates[1].microsecond)

        # set value via new datetime object
        today = datetime.today()
        obj.date = today
        self.assertEqual(obj.node.xpath('string(date)'), today.isoformat())


    def testFormattedDateTimeField(self):
        class TestObject(xmlmap.XmlObject):
            date = xmlmap.DateTimeField('date', format='%Y-%m-%dT%H:%M:%S')
            dates = xmlmap.DateTimeListField('date', format='%Y-%m-%dT%H:%M:%S')

        obj = TestObject(self.fixture)
        # fields should be datetime objects
        self.assert_(isinstance(obj.date, datetime))
        self.assert_(isinstance(obj.dates[1], datetime))
        # inspect date parsing
        self.assertEqual(2010, obj.date.year)
        self.assertEqual(1, obj.date.month)
        self.assertEqual(3, obj.date.day)
        self.assertEqual(2, obj.date.hour)
        self.assertEqual(13, obj.date.minute)
        self.assertEqual(44, obj.date.second)
        # microsecond date parsing
        self.assertEqual(3000, obj.dates[1].microsecond)

        # set value via new datetime object
        today = datetime.today()
        obj.date = today
        self.assertEqual(obj.node.xpath('string(date)'), today.strftime('%Y-%m-%dT%H:%M:%S')))


    def testSchemaField(self):
        # very simple xsd schema and valid/invalid xml based on the one from lxml docs:
        #   http://codespeak.net/lxml/validation.html#xmlschema
        xsd = '''<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
            <xsd:element name="a" type="AType"/>
            <xsd:complexType name="AType">
                <xsd:sequence>
                    <xsd:element name="b" type="BType" />
                </xsd:sequence>
            </xsd:complexType>
            <xsd:simpleType name="BType">
                <xsd:restriction base="xsd:string">
                    <xsd:enumeration value="c"/>
                    <xsd:enumeration value="d"/>
                    <xsd:enumeration value="e"/>
                </xsd:restriction>
            </xsd:simpleType><commit_msg>minor clean up on formatted date-time tests<commit_after># file test_xmlmap/test_fields.py
# 
#   Copyright 2011 Emory University Libraries
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

#!/usr/bin/env python

from datetime import datetime
import tempfile
import unittest

import eulxml.xmlmap.core as xmlmap
from testcore import main

class TestFields(unittest.TestCase):
    FIXTURE_TEXT = '''
        <foo id='a' xmlns:ex='http://example.com/'>
            <bar>
                <baz>42</baz>
            </bar>
            <bar>
                <baz>13</baz>
            </bar>
            <empty_field/>
            <boolean>
                    <text1>yes</text1>
                    <text2>no</text2>
                    <num1>1</num1>
                    <num2>0</num2>
            </boolean>
            <spacey>           this text
        needs to be
                normalized
            </spacey>
            <date>2010-01-03T02:13:44</date>
            <date>2010-01-03T02:13:44.003</date>
        </foo>
    '''

    namespaces = {'ex' : 'http://example.com/'}

    def setUp(self):
        # parseString wants a url. let's give it a proper one.
        url = '%s#%s.%s' % (__file__, self.__class__.__name__, 'FIXTURE_TEXT')

        self.fixture = xmlmap.parseString(self.FIXTURE_TEXT, url)
        
        self.rel_url = '%s#%s' % (__file__, self.__class__.__name__)

    def _empty_fixture(self):
        return xmlmap.parseString('<root/>', self.rel_url)

    def testvalidXpath(self):
        self.assertRaises(Exception, xmlmap.StringField, '["')
        
    def testNodeField(self):
        class TestSubobject(xmlmap.XmlObject):
            ROOT_NAME = 'bar'
            val = xmlmap.StringField('baz')

        class TestObject(xmlmap.XmlObject):
            child = xmlmap.NodeField('bar[1]', TestSubobject, required=True)
            missing = xmlmap.NodeField('missing', TestSubobject)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.child.val, '42')
        self.assertEqual(obj.missing, None)
        # undefined if >1 matched nodes

        # test set
        obj.child = TestSubobject(val='144')
        self.assertEqual(obj.child.val, '144')

        # check required
        self.assertTrue(obj._fields['child'].required)

        # test create_foo functionality
        class CreateTestObject(xmlmap.XmlObject):
            missing = xmlmap.NodeField('missing', TestSubobject)
        obj = CreateTestObject(self.fixture)
        self.assert_(obj.missing is None)
        obj.create_missing()
        self.assert_(isinstance(obj.missing, TestSubobject))

        # test del
        del obj.missing
        self.assertEqual(obj.missing, None)

        # test DEPRECATED instantiate on get hack
        class GetTestObject(xmlmap.XmlObject):
            missing = xmlmap.NodeField('missing', TestSubobject, instantiate_on_get=True)
        obj = GetTestObject(self.fixture)
        self.assert_(isinstance(obj.missing, TestSubobject),
            "non-existent nodefield is created on get when 'instantiate_on_get' flag is set")

    def testNodeListField(self):
        class TestSubobject(xmlmap.XmlObject):
            val = xmlmap.IntegerField('baz')

        class TestObject(xmlmap.XmlObject):
            children = xmlmap.NodeListField('bar', TestSubobject)
            missing = xmlmap.NodeListField('missing', TestSubobject, required=False)

        obj = TestObject(self.fixture)
        child_vals = [ child.val for child in obj.children ]
        self.assertEqual(child_vals, [42, 13])
        self.assertEqual(obj.missing, [])

        # check required
        self.assertFalse(obj._fields['missing'].required)

        # is_empty
        self.assertFalse(obj.children.is_empty())
        del obj.children
        self.assertTrue(obj.children.is_empty())


    def testStringField(self):
        class TestObject(xmlmap.XmlObject):
            val = xmlmap.StringField('bar[1]/baz', required=True)
            empty = xmlmap.StringField('empty_field', required=False)
            missing = xmlmap.StringField('missing')
            missing_ns = xmlmap.StringField('ex:missing')
            missing_att = xmlmap.StringField('@missing')
            missing_att_ns = xmlmap.StringField('@ex:missing')
            sub_missing = xmlmap.StringField('bar[1]/missing')
            multilevel_missing = xmlmap.StringField('missing_parent/missing_child')
            mixed = xmlmap.StringField('bar[1]')
            id = xmlmap.StringField('@id')
            spacey = xmlmap.StringField('spacey')
            normal_spacey = xmlmap.StringField('spacey', normalize=True)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.val, '42')
        self.assertEqual(obj.missing, None)
        # undefined if >1 matched nodes

        # access normalized and non-normalized versions of string field
        self.assertNotEqual("this text needs to be normalized", obj.spacey)
        self.assertEqual("this text needs to be normalized", obj.normal_spacey)

        # set an existing string value
        obj.val = "forty-two"
        # check that new value is set in the node
        self.assertEqual(obj.node.xpath('string(bar[1]/baz)'), "forty-two")
        # check that new value is accessible via descriptor        
        self.assertEqual(obj.val, 'forty-two')

        # set an attribute
        obj.id = 'z'
        # check that new value is set in the node
        self.assertEqual(obj.node.xpath('string(@id)'), "z")
        # check that new value is accessible via descriptor
        self.assertEqual(obj.id, 'z')

        # set value in an empty node
        obj.empty = "full"
        self.assertEqual(obj.node.xpath('string(empty_field)'), "full")
        # check that new value is accessible via descriptor
        self.assertEqual(obj.empty, 'full')

        # set missing fields
        obj.missing = 'not here'
        self.assertEqual(obj.node.xpath('string(missing)'), 'not here')
        # with ns
        obj.missing_ns = 'over there'
        self.assertEqual(obj.node.xpath('string(ex:missing)', namespaces=self.namespaces),
                    'over there')
        # in attrib
        obj.missing_att = 'out to pasture'
        self.assertEqual(obj.node.xpath('string(@missing)'), 'out to pasture')
        # in attrib with ns
        obj.missing_att_ns = "can't find me!"
        self.assertEqual(obj.node.xpath('string(@ex:missing)',
                        namespaces=self.namespaces), "can't find me!")
        # in subelement
        obj.sub_missing = 'pining (for the fjords)'
        self.assertEqual(obj.node.xpath('string(bar/missing)'), 'pining (for the fjords)')

        # in subelement which is itself missing
        obj.multilevel_missing = 'so, so gone'
        self.assertEqual(obj.node.xpath('string(missing_parent/missing_child)'), 'so, so gone')

        # attempting to set a node that contains non-text nodes - error
        self.assertRaises(Exception, obj.__setattr__, "mixed", "whoops")

        # set an existing string value to None
        # - attribute
        obj.id = None
        # check that attribute was removed from the xml
        self.assertEqual(0, obj.node.xpath('count(@id)'))
        self.assertEqual(None, obj.id)
        # - element
        obj.val = None
        # check that node is removed from the xml
        self.assertEqual(0, obj.node.xpath('count(bar[1]/baz)'))
        self.assertEqual(None, obj.val)

        # set to empty string - should store empty string and NOT remove the node
        # - attribute
        obj.id = ''
        # check that attribute was removed from the xml
        self.assertEqual('', obj.node.xpath('string(@id)'))
        self.assertEqual('', obj.id)
        # - element
        obj.val = ''
        # check that node is removed from the xml
        self.assertEqual('', obj.node.xpath('string(bar[1]/baz)'))
        self.assertEqual('', obj.val)

        # delete
        delattr(obj, 'spacey')
        # check that node is removed from the xml
        self.assertEqual(0, obj.node.xpath('count(spacey)'))
        self.assertEqual(None, obj.spacey)

        # check required
        self.assertTrue(obj._fields['val'].required)
        self.assertFalse(obj._fields['empty'].required)
        self.assertEqual(None, obj._fields['missing'].required)

    def testTextNode(self):
        # special case for text()
        class TextObject(xmlmap.XmlObject):
            text_node = xmlmap.StringField('text()')
            nested_text =  xmlmap.StringField('nest[@type="feather"]/text()')
            missing_text = xmlmap.StringField('missing[@type="foo"]/bar/text()')
            m_nested_attr = xmlmap.StringField('missing[@type="foo"]/bar/@label')
            m_nested_text = xmlmap.StringField('missing[@type="foo"]/baz')
            nested_child_pred = xmlmap.StringField('missing[@type="foo"][baz="bah"]/txt')
        # parseString wants a url. let's give it a proper one.
        url = '%s#%s.%s' % (__file__, self.__class__.__name__, 'TEXT_XML_FIXTURES')
        xml = '''<text>some text<nest type='feather'>robin</nest></text>'''
        obj = TextObject(xmlmap.parseString(xml, url))

        self.assertEqual('some text', obj.text_node)
        
        # set text node
        obj.text_node = 'la'
        self.assertEqual('la', obj.node.xpath('string(text())'))
        self.assertEqual('la', obj.text_node)
        # set to empty string
        obj.text_node = ''
        self.assertEqual('', obj.node.xpath('string(text())'))
        self.assertEqual('', obj.text_node)
        # set to value then set to None - text should be cleared out
        obj.text_node = 'la'
        obj.text_node = None
        self.assertEqual('', obj.node.xpath('string(text())'))
        self.assertEqual('', obj.text_node)

        self.assertEqual('robin', obj.nested_text)

        # create parent of a text node
        obj.missing_text = 'tra'
        self.assertEqual('tra', obj.node.xpath('string(missing/bar/text())'))
        self.assertEqual('tra', obj.missing_text)

    def testStringListField(self):
        class TestObject(xmlmap.XmlObject):
            vals = xmlmap.StringListField('bar/baz', required=True)
            missing = xmlmap.StringListField('missing')
            spacey = xmlmap.StringListField('spacey', normalize=True)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.vals, ['42', '13'])
        self.assertEqual(obj.missing, [])

        # access normalized string list field
        self.assertEqual("this text needs to be normalized", obj.spacey[0])

        # delete list field
        delattr(obj, 'spacey')
        # check that node is removed from the xml
        self.assertEqual(0, obj.node.xpath('count(spacey)'))
        self.assertEqual([], obj.spacey)

        # check required
        self.assertTrue(obj._fields['vals'].required)
        

    def testIntegerField(self):
        class TestObject(xmlmap.XmlObject):
            val = xmlmap.IntegerField('bar[2]/baz', required=True)
            count = xmlmap.IntegerField('count(//bar)')
            missing = xmlmap.IntegerField('missing')
            nan = xmlmap.IntegerField('@id')

        obj = TestObject(self.fixture)
        self.assertEqual(obj.val, 13)
        self.assertEqual(obj.count, 2)
        self.assertEqual(obj.missing, None)
        self.assertEqual(obj.nan, None)
        # undefined if >1 matched nodes

        # set an integer value
        obj.val = 14
        # check that new value is set in the node
        self.assertEqual(obj.node.xpath('number(bar[2]/baz)'), 14)
        # check that new value is accessible via descriptor
        self.assertEqual(obj.val, 14)

        # check required
        self.assertTrue(obj._fields['val'].required)

    def testIntegerListField(self):
        class TestObject(xmlmap.XmlObject):
            vals = xmlmap.IntegerListField('bar/baz')
            missing = xmlmap.IntegerListField('missing', required=False)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.vals, [42, 13])
        self.assertEqual(obj.missing, [])

        # check required
        self.assertFalse(obj._fields['missing'].required)

    def testItemField(self):
        class TestObject(xmlmap.XmlObject):
            letter = xmlmap.ItemField('substring(bar/baz, 1, 1)')
            missing = xmlmap.ItemField('missing', required=False)

        obj = TestObject(self.fixture)
        self.assertEqual(obj.letter, '4')
        self.assertEqual(obj.missing, None)

        # check required
        self.assertFalse(obj._fields['missing'].required)

    def testBooleanField(self):        
        class TestObject(xmlmap.XmlObject):
            txt_bool1 = xmlmap.SimpleBooleanField('boolean/text1', 'yes', 'no', required=False)
            txt_bool2 = xmlmap.SimpleBooleanField('boolean/text2', 'yes', 'no')
            num_bool1 = xmlmap.SimpleBooleanField('boolean/num1', 1, 0)
            num_bool2 = xmlmap.SimpleBooleanField('boolean/num2', 1, 0)
            opt_elem_bool = xmlmap.SimpleBooleanField('boolean/opt', 'yes', None)
            opt_attr_bool = xmlmap.SimpleBooleanField('boolean/@opt', 'yes', None)

        #obj = TestObject(self.fixture.documentElement)
        obj = TestObject(self.fixture)
        self.assertEqual(obj.txt_bool1, True)
        self.assertEqual(obj.txt_bool2, False)
        self.assertEqual(obj.num_bool1, True)
        self.assertEqual(obj.num_bool2, False)
        self.assertEqual(obj.opt_elem_bool, False)
        self.assertEqual(obj.opt_attr_bool, False)

        # set text boolean
        obj.txt_bool1 = False
        # check that new value is set in the node
        self.assertEqual(obj.node.xpath('string(boolean/text1)'), 'no')
        # check that new value is accessible via descriptor
        self.assertEqual(obj.txt_bool1, False)

        # set numeric boolean
        obj.num_bool1 = False
        # check for new new value in the node and via descriptor
        self.assertEqual(obj.node.xpath('number(boolean/num1)'), 0)
        self.assertEqual(obj.num_bool1, False)

        # set optional element boolean
        obj.opt_elem_bool = True
        self.assertEqual(obj.node.xpath('string(boolean/opt)'), 'yes')
        self.assertEqual(obj.opt_elem_bool, True)
        obj.opt_elem_bool = False
        self.assertEqual(obj.node.xpath('count(boolean/opt)'), 0)
        self.assertEqual(obj.opt_elem_bool, False)

        # set optional attribute boolean
        obj.opt_attr_bool = True
        self.assertEqual(obj.node.xpath('string(boolean/@opt)'), 'yes')
        self.assertEqual(obj.opt_attr_bool, True)
        obj.opt_attr_bool = False
        self.assertEqual(obj.node.xpath('count(boolean/@opt)'), 0)
        self.assertEqual(obj.opt_attr_bool, False)

        # check required
        self.assertFalse(obj._fields['txt_bool1'].required)


    # FIXME: DateField and DateListField are hacked together. Until we
    #   work up some proper parsing and good testing for them, they should
    #   be considered untested and undocumented features.

    
    def testDateTimeField(self):        
        class TestObject(xmlmap.XmlObject):
            date = xmlmap.DateTimeField('date')
            dates = xmlmap.DateTimeListField('date')

        obj = TestObject(self.fixture)
        # fields should be datetime objects
        self.assert_(isinstance(obj.date, datetime))
        self.assert_(isinstance(obj.dates[1], datetime))
        # inspect date parsing
        self.assertEqual(2010, obj.date.year)
        self.assertEqual(1, obj.date.month)
        self.assertEqual(3, obj.date.day)
        self.assertEqual(2, obj.date.hour)
        self.assertEqual(13, obj.date.minute)
        self.assertEqual(44, obj.date.second)
        # microsecond date parsing
        self.assertEqual(3000, obj.dates[1].microsecond)

        # set value via new datetime object
        today = datetime.today()
        obj.date = today
        self.assertEqual(obj.node.xpath('string(date)'), today.isoformat())


    def testFormattedDateTimeField(self):
        class TestObject(xmlmap.XmlObject):
            date = xmlmap.DateTimeField('date', format='%Y-%m-%dT%H:%M:%S')
            dates = xmlmap.DateTimeListField('date', format='%Y-%m-%dT%H:%M:%S.%f')

        obj = TestObject(self.fixture)
        # fields should be datetime objects
        self.assert_(isinstance(obj.date, datetime))
        self.assert_(isinstance(obj.dates[1], datetime))
        # inspect date parsing
        self.assertEqual(2010, obj.date.year)
        self.assertEqual(1, obj.date.month)
        self.assertEqual(3, obj.date.day)
        self.assertEqual(2, obj.date.hour)
        self.assertEqual(13, obj.date.minute)
        self.assertEqual(44, obj.date.second)
        # microsecond date parsing
        self.assertEqual(3000, obj.dates[1].microsecond)

        # set value via new datetime object
        today = datetime.today()
        obj.date = today
        self.assertEqual(obj.node.xpath('string(date)'), today.strftime('%Y-%m-%dT%H:%M:%S'))


    def testSchemaField(self):
        # very simple xsd schema and valid/invalid xml based on the one from lxml docs:
        #   http://codespeak.net/lxml/validation.html#xmlschema
        xsd = '''<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
            <xsd:element name="a" type="AType"/>
            <xsd:complexType name="AType">
                <xsd:sequence>
                    <xsd:element name="b" type="BType" />
                </xsd:sequence>
            </xsd:complexType>
            <xsd:simpleType name="BType">
                <xsd:restriction base="xsd:string">
                    <xsd:enumeration value="c"/>
                    <xsd:enumeration value="d"/>
                    <xsd:enumeration value="e"/>
                </xsd:restriction>
            </xsd:simpleType>
{'fraction_non_alphanumeric': 0.12091592245698105, 'fraction_numerical': 0.009202788063602701, 'mean_word_length': 2.426212686567164, 'pattern_counts': {'":': 0, '<': 95, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 168}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5606472', 'n_tokens_mistral': 10872, 'n_tokens_neox': 10380, 'n_words': 2309}