<commit_before>"""
Project: META-SHARE prototype implementation
 Author: <NAME> <<EMAIL>>
"""
import logging
import re

from django.core.cache import cache
from django.core.exceptions import ValidationError, ObjectDoesNotExist, \
  ImproperlyConfigured
from django.db.models.fields import related
from django.db import models, IntegrityError
from traceback import format_exc
from xml.etree.ElementTree import Element, fromstring, tostring

from metashare.repository.fields import MultiSelectField, MultiTextField, \
  MetaBooleanField, DictField

from metashare.settings import LOG_LEVEL, LOG_HANDLER, \
  CHECK_FOR_DUPLICATE_INSTANCES
from django.db.models.fields.related import ForeignRelatedObjectsDescriptor, \
    OneToOneField
from Queue import Queue
from metashare.storage.models import MASTER

# Setup logging support.
logging.basicConfig(level=LOG_LEVEL)
LOGGER = logging.getLogger('metashare.repository.supermodel')
LOGGER.addHandler(LOG_HANDLER)

# cfedermann: this prevents a bug with PostgreSQL databases and Django 1.3
try:
    from django.db.backends.postgresql_psycopg2.base import DatabaseFeatures
    DatabaseFeatures.can_return_id_from_insert = True

except ImproperlyConfigured:
    pass

REQUIRED = 1
OPTIONAL = 2
RECOMMENDED = 3

# template of a META-SHARE metadata XML schema URL
SCHEMA_URL = 'http://metashare.ilsp.gr/META-XMLSchema/v{0}/' \
  'META-SHARE-Resource.xsd'

METASHARE_ID_REGEXP = re.compile('<metashareId>.+</metashareId>',
  re.I|re.S|re.U)

OBJECT_XML_CACHE = {}

# pylint: disable-msg=W0611
from metashare import repository

def _remove_namespace_from_tags(element_tree):
    """
    Removes any namespace information from the given element_tree instance.
    """
    element_tree.tag = element_tree.tag.split('}')[-1]
    
    for child in element_tree.getchildren():
        _remove_namespace_from_tags(child)
    
    return element_tree

def _classify(class_name):
    """
    Converts the given class name into the corresponding class type object.
    """
    try:
        return eval('repository.models.{}'.format(class_name))

    except NameError:
        return None


def _make_choices_from_list(source_list):
    """
    Converts a given list of Strings to tuple choices.

    Returns a dictionary containing two keys:
    - max_length: the maximum char length for this source list,
    - choices: the list of (index, value) tuple choices.
    """
    _choices = []
    for index, value in enumerate(source_list):
        _choices.append((str(index), value))
    return {'max_length': len(_choices)/10+1, 'choices': tuple(_choices)}

def _make_choices_from_int_list(source_list):
    """
    Converts a given list of Integers to tuple choices.

    Returns a dictionary containing two keys:
    - max_length: the maximum char length for this source list,
    - choices: the list of (value, value) tuple choices.
    """
    _choices = []
    for value in source_list:
        _choices.append((value, value))
    return {'max_length': len(_choices)/10+1, 'choices': tuple(_choices)}


class SchemaModel(models.Model):
    """
    Super class for all XSD schema types/components.
    """
    __schema_name__ = 'UNDEFINED'
    __schema_fields__ = ()
    __schema_attrs__ = ()
    __schema_classes__ = {}
    __schema_parent__ = None
    
    class Meta:
        """
        This is an abstract super class for all schema models.
        """
        abstract = True

    @classmethod
    def is_required_field(cls, name):
        """
        Checks whether the field with the given name is a required field.
        """
        # pylint: disable-msg=E1101
        _fields = cls.get_fields()
        return name in _fields['required']

    @classmethod
    def get_many_to_many_fields(cls):
        """
        Returns a list containing all ManyToManyField fields for this model.
        """
        return [field.name for field in cls._meta.local_many_to_many]

    @classmethod
    def get_fields_flat(cls, unique_values=True):
        """
        Return all fields in a flat list
        """
        _fields = []
        for _, _field, _ in cls.__schema_fields__:
            if unique_values and _field in _fields:
                continue
            _fields.append(_field)

        return _fields

    @classmethod
    def get_fields(cls, unique_values=True):
        """
        Returns a dictionary containing all fields.
        """
        _fields = {'required': [], 'recommended': [], 'optional': []}
        for _not_used, _field, _required in cls.__schema_fields__:
            if _required == REQUIRED:
                if unique_values and _field in _fields['required']:
                    continue

                _fields['required'].append(_field)

            elif _required == RECOMMENDED:
                if unique_values and _field in _fields['recommended']:
                    continue

                _fields['recommended'].append(_field)

            else:
                if unique_values and _field in _fields['optional']:
                    continue

                _fields['optional'].append(_field)

        return _fields

    @classmethod
    def get_field_sets(cls, unique_values=True):
        """
        Returns a dictionary containing just "*_set" fields.
        """
        _field_sets = {'required': [], 'recommended': [], 'optional': []}
        for _not_used, _field, _required in cls.__schema_fields__:
            if _field.endswith('_set'):
                if _required == REQUIRED:
                    if unique_values and _field in _field_sets['required']:
                        continue

                    _field_sets['required'].append(_field)

                elif _required == RECOMMENDED:
                    if unique_values and _field in _field_sets['recommended']:
                        continue

                    _field_sets['recommended'].append(_field)

                else:
                    if unique_values and _field in _field_sets['optional']:
                        continue

                    _field_sets['optional'].append(_field)

        return _field_sets

    @classmethod
    def get_verbose_name(cls, fieldname):
        """
        For a real field such as 'firstName'
        or a pseudo-field such as 'contactinfotype_model_set',
        obtain the best possible verbose name that we can give.
        """
        #fields = cls._meta.get_field_by_name(fieldname)
        #if fields:
        #    field = fields[0]
        #    return field.verbose_name
        try:
            field = cls._meta.get_field(fieldname)
            return field.verbose_name
        except:
            remote = getattr(cls, fieldname, None)
            if isinstance(remote, ForeignRelatedObjectsDescriptor):
                return remote.related.model._meta.verbose_name
            return fieldname

    @staticmethod
    def _python_to_xml(value):
        """
        Converts the given Python value to its XML representation.
        """
        # Boolean values need to be rendered as 'true' or 'false' Strings.
        if isinstance(value, bool):
            if value:
                return 'true'
            else:
                return 'false'

        # String instances are converted to Unicode instances.
        elif isinstance(value, basestring):
            return unicode(value)

        # All other values are encoded as Strings.
        else:
            return str(value)

    @staticmethod
    def _xml_to_python(value, field):
        """
        Converts the given XML value to its Python representation for field.
        """
        # If no field is given, we have to return the value unmodified.
        if field is None:
            return value

        result = value

        # Handle fields with choices as we need to convert the current value
        # to the database representation value of the respective choice.
        if len(field.choices) > 0:
            # MetaBooleanField instances are a special case: here, we need to
            # return either 'Yes' or 'No', depending on the given value.
            if isinstance(field, MetaBooleanField):
                if value.strip().lower() in ('true', 'yes', '1'):
                    result = 'Yes'

                elif value.strip().lower() in ('false', 'no', '0'):
                    result = 'No'
                
                else:
                    LOGGER.error(u'Value {} not a valid MetaBooleanField ' \
                      'choice for {}'.format(repr(value), field.name))

            else:
                for _db_value, _readable_value in field.choices:
                    if _readable_value == value:
                        result = _db_value

                if result == value \
                        and not isinstance(field, models.IntegerField):
                    LOGGER.error(u'Value {} not found in choices for ' \
                      '{}'.format(repr(value), field.name))

        elif isinstance(field, models.BooleanField) \
          or isinstance(field, models.NullBooleanField):
            if value.strip().lower() == 'true':
                result = True

            elif value.strip().lower() == 'false':
                result = False

            else:
                LOGGER.error(u'Value {} not a valid Boolean for {}'.format(
                  repr(value), field.name))

        elif isinstance(field, models.TextField) \
          or isinstance(field, MultiTextField):
            if value is None:
                result = ''

        return result

    def export_to_elementtree(self):
        """
        Exports this instance to an XML ElementTree.
        """
        if self.__schema_name__ == "SUBCLASSABLE":
            # pylint: disable-msg=E1101
            return self.as_subclass().export_to_elementtree()

        _root = Element(self.__schema_name__)

        # Fix namespace attributes for the resourceInfo root element.
        if _root.tag == 'resourceInfo':
            # only import on demand (metashare.repository.models depends on
            # metashare.repository.supermodel, giving a circular dependency if
            # importin 'globally')
            from metashare.repository.models import SCHEMA_NAMESPACE, \
                SCHEMA_VERSION
            _root.attrib['xmlns'] = SCHEMA_NAMESPACE
            _root.attrib['xmlns:xsi'] = "http://www.w3.org/2001/" \
              "XMLSchema-instance"
            _root.attrib['xsi:schemaLocation'] = "{} {}" \
              .format(SCHEMA_NAMESPACE, SCHEMA_URL.format(SCHEMA_VERSION))

        # We first serialize all schema attributes.  For the moment, we just
        # assume that attributes are Strings only.  This holds for the v1.1
        # representative XSD, still more clever value handling might actually
        # be required for version v2; depends on the final schema...
        for _xsd_attr, _model_field, _not_used in self.__schema_attrs__:
            try:
                # First, we check if there is a get_FOO_display() method.
                _func = getattr(self, 'get_{0}_display'.format(_model_field))

                # If so, we call this method to determine the current value.
                _value = _func()

            except AttributeError:
                # Otherwise, we try to retrieve the value via getattr().
                _value = getattr(self, _model_field, None)

            # If a value could be found, it becomes an attribute of the node.
            if _value is not None:
                _root.attrib[_xsd_attr] = SchemaModel._python_to_xml(_value)

        #print self.get_fields()
        #print self.get_field_sets()

        # Then, we loop over all schema fields, retrieve their values and put
        # XML-ified versions of these values into the XML tree.
        for _xsd_field, _model_field, _not_used in self.__schema_fields__:
            try:
                # Again, we first check for a get_FOO_display() method.
                _func = getattr(self, 'get_{0}_display'.format(_model_field))
                _value = _func()

            except AttributeError:
                # Only falling back to getattr() if it doesn't exist.
                _value = getattr(self, _model_field, None)

            if _value is not None and _value != "":
                _model_set_value = _model_field.endswith('_model_set')

                if not _model_set_value:
                    _field = self._meta.get_field_by_name(_model_field)[0]
                else:
                    _field = None

                # For MetaBooleanFields, we actually need the database value
                # (i.e., not the displayed value).
                if isinstance(_field, MetaBooleanField):
                    _value = getattr(self, _model_field, [])

                # For MultiSelectFields, we call get_FOO_display_list().
                elif isinstance(_field, MultiSelectField):
                    _func = getattr(self, 'get_{0}_display_list'.format(
                      _model_field))
                    _value = _func()
                    
                    # Sort MultiSelectField values to allow comparison!
                    _value.sort()

                # For DictFields, we convert the dict to a list of tuples.
                elif isinstance(_field, DictField):
                    _value = _value.items()

                # For ManyToManyFields, compute all related objects.
                if isinstance(_value, models.Manager):
                    _value = _value.all().order_by('id')

                # If the value is not yet of list type, we wrap it in a list.
                elif not isinstance(_value, list):
                    _value = [_value]

                # Iterate over all values inside the current _value list.
                for _sub_value in _value:
                    # For SubclassableModel instances, we have to check if the
                    # sub class type matches the current XSD name; otherwise
                    # we have to ignore the current value.
                    if isinstance(_sub_value, SubclassableModel):
                        _clazz = _sub_value.as_subclass()
                        _clazz_name = _clazz.__class__.__name__
                        _choice_name = None
                        _xsd_name = _xsd_field.split('/')[-1]

                        if _xsd_name in self.__schema_classes__.keys():
                            _choice_name = self.__schema_classes__[_xsd_name]

                        if _clazz_name != _choice_name:
                            LOGGER.debug(u'Skipping choice value {}'.format(
                              _xsd_name))
                            continue

                        # If this is the correct sub class type, copy _clazz
                        # to the _sub_value variable for further processing...
                        _sub_value = _clazz

                    # Handle element paths inside _xsd_field names.  Supports
                    # element paths such as e.g. affiliation/OrganizationInfo.
                    _current_node = _root
                    _path = _xsd_field.split('/')
                    if len(_path) > 1:
                        # Iterate over all intermediate names and then create
                        # the corresponding element for these.
                        for _tag in _path[:-1]:
                            # Only create a new intermediate element if there
                            # is not already one available...
                            _element = None
                            if _model_set_value:
                                _element = _current_node.find(_tag)

                            if _element is None:
                                _element = Element(_tag)
                                _current_node.append(_element)

                            _current_node = _element

                    # The last name inside the path becomes our _xsd_name.
                    _xsd_name = _path[-1]

                    # For complex values, we recursively call this method.
                    if isinstance(_sub_value, SchemaModel):
                        if _sub_value.__schema_name__ == "STRINGMODEL":
                            _element = Element(_xsd_name)
                            _element.text = SchemaModel._python_to_xml(_sub_value.value)
                            _current_node.append(_element)

                        else:
                            _sub_value = _sub_value.export_to_elementtree()

                            # We fix the sub value's tag as it may be "wrong".
                            # E.g., PersonInfo is sometimes called contactPerson.
                            _sub_value.tag = _xsd_name

                            # And append the sub structure to the current node.
                            _current_node.append(_sub_value)

                    # Simple values are added to a new element and appended.
                    else:
                        _element = Element(_xsd_name)
                        if isinstance(_sub_value, tuple):
                            # tuple values come from DictFields with an RFC 3066
                            # language code key
                            _element.set('lang', _sub_value[0])
                            _element.text = SchemaModel._python_to_xml(
                                                                _sub_value[1])                            
                            if self.is_required_field(_model_field):
                                # If the element is "required" in the model,
                                # turn the "required" value of the element
                                # tree to "1" (true), otherwise "0" (false).
                                _element.required = self.is_required_field(
                                                                _model_field)
                            else:
                                _element.required = False
                        else:
                            _element.text = SchemaModel._python_to_xml(
                                                                _sub_value)                            
                            if self.is_required_field(_model_field):
                                _element.required = self.is_required_field(
                                                                _model_field)
                            else:
                                _element.required = False
                        _current_node.append(_element)

        # Return root node of the ElementTree; can be converted to String
        # using: xml.etree.ElementTree.tostring(_root, encoding="utf-8")
        return _root

    @classmethod
    def _check_for_duplicates(cls, _object):
        """
        Checks if the given _object is a redundant copy of another object.

        Returns a list containing all existing objects that are equal to the
        given _object instance, sorted by primary key 'id'.

        """
        if not CHECK_FOR_DUPLICATE_INSTANCES:
            return []

        _was_duplicate = False
        _related_objects = []

        # We collect all value constraints in a dictionary.
        kwargs = {}

        # Build list of field names for this class type.  Using introspection
        # with cls._meta.get_all_field_names() would not work as it contains
        # additional fields such as id and related fields that would break our
        # comparison code.
        _fields = [x[1] for x in cls.__schema_fields__]
        _fields.extend([x[1] for x in cls.__schema_attrs__])
        if hasattr(_object, 'copy_status'):
            _fields.append('copy_status')
        
        if _object.__schema_name__ == "STRINGMODEL":
            _fields.append("value")
        
        # Ensure that "back_to_" pointers are available for checking.
        for _field in cls._meta.fields:
            if isinstance(_field, related.ForeignKey):
                if _field.name.startswith('back_to_') and \
                  not _field.name in _fields:
                    _fields.append(_field.name)

        # We iterate over all model fields of the current _object.
        for field_name in _fields:
            # OneToMany fields need to be handled later, hence we add these to
            # our _related_objects list and continue with the loop.
            if field_name.endswith('_set'):
                _related_objects.append(field_name)
                LOGGER.debug(u'Skipping OneToMany "{}"'.format(field_name))
                continue

            # Retrieve the model field value using getattr().  This is enough
            # as even if a get_FOO_display() method would exist for a field,
            # the "raw" values needed to be compared anyway.
            _value = getattr(_object, field_name, None)

            # Retrieve model field instance to allow type checking.
            _field = _object._meta.get_field_by_name(field_name)[0]

            # For ManyToManyFields, compute the list of related objects.
            if isinstance(_field, related.ManyToManyField):
                _value = _value.all()

                # If there is no related object for this ManyToManyField, we
                # have to constrain kwargs[field_name] with the None value.
                if not len(_value):
                    _value = None

                # We have to use '__in' QuerySet lookup to avoid the "more
                # than one row returned by a subquery used as an expression"
                # bug which raises a DatabaseError otherwise.

                else:
                    field_name += '__in'

            # Finally, OneToOneField instances have to be checked as related
            # objects and hence need to be put into _related_objects.
            elif isinstance(_field, related.OneToOneField):
                LOGGER.debug(u'Skipping OneToOne "{}"'.format(field_name))
                _related_objects.append(field_name)
                continue

            # For ForeignKey instances, we have to check their name first.
            # "back_to_" fields need to be put into _related_objects, while
            # "normal" ForeignKey fields can be checked with our QuerySet.
            elif isinstance(_field, related.ForeignKey):
                if field_name.startswith('back_to_'):
                    LOGGER.debug(u'Skipping back_to "{}"'.format(field_name))
                    _related_objects.append(field_name)
                    continue

            # If the field value is not None or the field allows None as value
            # we set the current value as constraint in our kwargs dictionary.
            if _value is not None:
                kwargs[field_name] = _value

            elif _field.null:
                kwargs[field_name] = None

            # Otherwise, we print a message that we skip the current field.
            else:
                LOGGER.debug(u'Skipping {0}={1} ({2})'.format(field_name,
                  _value, type(_field)))

        # Use **magic to create a constrained QuerySet from kwargs.
        query_set = cls.objects.filter(**kwargs).order_by('id')

        _duplicates = []
        if query_set.count() > 1:
            # We now know that there may exist at least one duplicate for the
            # given _object;  we have to check the related objects to be sure.
            
            # Convert the current object into its serialised XML String and
            # remove any META-SHARE related id from this String.
            cache_key = '{}_{}'.format(type(_object).__name__.lower(),
              _object.id)
            if OBJECT_XML_CACHE.has_key(cache_key):
                _obj_value = OBJECT_XML_CACHE[cache_key]
            
            else:
                _obj_value = tostring(_object.export_to_elementtree())
                _obj_value = METASHARE_ID_REGEXP.sub('', _obj_value)
                OBJECT_XML_CACHE[cache_key] = _obj_value

            # Iterate over all potential duplicates, ordered by increasing id.
            for _candidate in query_set.iterator():
                # Skip the current candidate if it is our _object itself.
                if _candidate == _object:
                    continue

                # Convert candidate into XML String and remove META-SHARE id.
                cache_key = '{}_{}'.format(type(_candidate).__name__.lower(),
                  _candidate.id)
                if OBJECT_XML_CACHE.has_key(cache_key):
                    _check = OBJECT_XML_CACHE[cache_key]

                else:
                    _check = tostring(_candidate.export_to_elementtree())
                    _check = METASHARE_ID_REGEXP.sub('', _check)
                    OBJECT_XML_CACHE[cache_key] = _check

                # If both XML Strings are equal, we have found a duplicate!
                if _obj_value == _check:
                    _duplicates.append(_candidate)

        return _duplicates

    @staticmethod
    def _cleanup(objects, only_remove_duplicates=False):
        """
        Deletes all objects within the given objects list.

        The objects list contains tuples (obj, status) where obj is a Django
        object instance and status is in {'C', 'D', 'O'}.

        * 'C' == 'Created':   The corresponding object has just been created.

        * 'D' == 'Duplicate': The corresponding object is a duplicate.

        * 'O' == 'Original':  The corresponding object pre-existed in the DB.

        If only_remove_duplicates=True, only object instances with status='D'
        will be deleted while other instances are kept.
        
        Returns a list containing all instances which have not been deleted.

        """
        _objects = []
        for (obj, status) in objects:
            if only_remove_duplicates and status != 'D':
                LOGGER.debug(u'Keeping object {} ({})'.format(obj, status))
                _objects.append((obj, status))
                continue

            if obj.id:
                try:
                    LOGGER.debug(u'Deleting object {0}'.format(obj))
                    cache_key = '{}_{}'.format(type(obj).__name__.lower(),
                      obj.id)

                    if OBJECT_XML_CACHE.has_key(cache_key):
                        OBJECT_XML_CACHE.pop(cache_key)

                    if obj.__schema_name__ == "resourceInfo":
                        storage_object = obj.storage_object
                        storage_object.delete()
                    
                    obj.delete()             

                except ObjectDoesNotExist:
                    continue
        
        return _objects

    # pylint: disable-msg=R0911
    @classmethod
    def import_from_elementtree(
      cls, element_tree, cleanup=True, parent=None, copy_status=MASTER):
        """
        Imports the given XML ElementTree into an instance of type cls.

        Returns a tuple containing a reference to the created object instance
        as first value and the list of all objects that have been created when
        import the given XML ElementTree as second value.

        Returns (None, [], error_msg) in case of errors.
        """
        LOGGER.debug(u'parent: {0}'.format(parent))
        
        # We ignore name space information in tags, hence we remove it.
        element_tree = _remove_namespace_from_tags(element_tree)

        # First, we make sure that the given element_tree has the right tag.
        if element_tree is None \
          or element_tree.tag != cls.__schema_name__:
            _msg = u"Tags don't match: {}!={}".format(element_tree.tag,
              cls.__schema_name__)
            LOGGER.error(_msg)
            return (None, [], _msg)

        # We collect a list of all objects that are (recursively) created by
        # the importing process.  This is required if rollback has to be done.
        _created = []

        # We also need to instantiate a new instance of this class type.
        _object = cls()
        
        if hasattr(_object, 'copy_status'):
            _object.copy_status = copy_status

        # If a parent instance is given, add it to the object instance.
        if parent:
            _class_name = parent.__class__.__name__.lower()
            _foreign_key_name = 'back_to_{0}'.format(_class_name)

            try:
                LOGGER.debug(u'Setting schema parent {0}={1}'.format(
                  _foreign_key_name, parent))

                _ = _object._meta.get_field_by_name(_foreign_key_name)[0]
                setattr(_object, _foreign_key_name, parent)

            except (models.FieldDoesNotExist, AttributeError) as _exc:
                _msg = u"Could not set schema parent! ({})".format(_exc)
                LOGGER.error(_msg)
                SchemaModel._cleanup(_created)
                return (None, [], _msg)

        for xsd_attr, _model_field, _not_used in cls.__schema_attrs__:
            # Get attribute value and field instance for this xsd_attr.
            _attr = element_tree.attrib.get(xsd_attr, None)
            _field = _object._meta.get_field_by_name(_model_field)[0]

            # Convert attribute value to its Python representation for field.
            _value = SchemaModel._xml_to_python(_attr, _field)

            # If a value could be found, set the model field value.
            if _value is not None:
                setattr(_object, _model_field, _value)

        # Iterate over all model fields for an cls instance.  These need to be
        # sorted so that any _set fields are processed at the end!
        _fields_before_sets = list(cls.__schema_fields__)

        # The following allows to sort all ManyToMany field instances to the
        # end of the list; this is required to ensure that all required fields
        # can be handled and filled before saving this instance for handling
        # the ManyToMany fields...
        _fields_before_sets.sort(key=lambda x: x[2] == REQUIRED, reverse=True)

        _many_to_many_fields = []
        for _field in _fields_before_sets:
            if _field[1].endswith('_set'):
                continue

            _model_field = _object._meta.get_field_by_name(_field[1])[0]
            if isinstance(_model_field, related.ManyToManyField):
                _many_to_many_fields.append(_field)
                _fields_before_sets.remove(_field)

        _fields_before_sets.extend(_many_to_many_fields)

        _fields_before_sets.sort(cmp=lambda x, y: -y[1].endswith('_set'))

        for xsd_field, _model_field, _required in _fields_before_sets:
            # We want to collect all values for the current model field.  This
            # may be several, e.g., for multiple contactPerson elements.
            # Language tags currently only exist for DictFields. If the language
            # tags list is not empty, then its indexes will always correspond to
            # the indexes of the values list. 
            _values = []
            _lang_tags = []
            _parent = None

            # If we are handling a field set, we have to make sure that our
            # object instance has a valid identifier to be used as parent.
            # If the object instance's id is None, we have to save it first!
            if _model_field.endswith('_set'):
                if _object.id is None:
                    try:
                        LOGGER.debug(u'Saving parent object for {0}'.format(
                          _model_field))
                        _object.save()

                    except IntegrityError as _exc:
                        # pylint: disable-msg=E1101
                        _msg = u'Could not save {} object! ({})'.format(
                          _object.__class__, _exc)
                        LOGGER.error(_msg)

                        LOGGER.error(u'cls.__schema_parent__: {0}'.format(
                          cls.__schema_parent__))
                        LOGGER.error(u'parent: {0}'.format(parent))
                        if parent:
                            LOGGER.error(u'parent.__class__.__name__: ' \
                              u'{0}'.format(parent.__class__.__name__))

                        SchemaModel._cleanup(_created)
                        return (None, [], _msg)

                # Set current object instance as parent for recursive import.
                _parent = _object

            # Iterate over all sub elements with the current XSD field name.
            for _value in element_tree.findall(xsd_field):
                # As field sets are virtual, we set _field=None for them.
                if _model_field.endswith('_set'):
                    _field = None

                else:
                    _field = _object._meta.get_field_by_name(_model_field)[0]

                # The initial assumption is that duplicate objects should
                # be deleted immediately;  this does not hold for OneToOne
                # and OneToMany fields (where _parent is not None), only.
                _delete_duplicate_objects = True
                
                if isinstance(_field, related.OneToOneField) or _parent:
                    _delete_duplicate_objects = False

                # If the current value is simple-typed, append its text value.
                if not len(_value.getchildren()):
                    _text = SchemaModel._xml_to_python(_value.text, _field)
                    LOGGER.debug(u'_value.tag: {}, _value.text: {}'.format(
                      _value.tag, _text))

                    # We skip empty, simple-typed elements.
                    if not _text:
                        continue

                    # If the current field is a DictField, then we do not only
                    # have to fill the _values list
                    if isinstance(_field, DictField):
                        # 'und' is the ISO 639-2 special language code for an
                        # undetermined language
                        _lang_tags.append(_value.get('lang', 'und'))

                    elif _value.tag in cls.__schema_classes__.keys():
                        LOGGER.debug(u'_schema_classes__[{}] = {}'.format(
                          _value.tag, cls.__schema_classes__[_value.tag]))
                        _sub_cls = _classify(
                          cls.__schema_classes__[_value.tag])
                        if _sub_cls.__schema_name__ == "STRINGMODEL":
                            LOGGER.debug(u'Creating STRINGMODEL {}.'.format(
                              _text))
                            _instance = _sub_cls()
                            _instance.value = _text
                            _instance.save()

                            _duplicates = _sub_cls._check_for_duplicates(
                              _instance)
                            _was_duplicate = len(_duplicates) > 0

                            # Add current _instance instance to the _created
                            # list with correct status: 'D' if it was a
                            # duplicate, 'C' otherwise.
                            if _was_duplicate:
                                _created.append((_instance, 'D'))

                                # If we are allowed to perform cleanup, we do
                                # so and also replace our _instance instance
                                # with the "original" object.
                                if _delete_duplicate_objects:
                                    _created = SchemaModel._cleanup(_created,
                                      only_remove_duplicates=True)

                                    # Replace _instance with "original".
                                    _instance = _duplicates[0]
                                    _created.append((_instance, 'O'))

                            else:
                                _created.append((_instance, 'C'))

                            _text = _instance

                    _values.append(_text)

                # Otherwise, we have to handle the complex structure of value.
                else:
                    # For this, we have defined a mapping from XSD element
                    # names to Django model classes inside __schema_classes__.
                    if not _value.tag in cls.__schema_classes__.keys():
                        _msg = u'Unsupported tag: {} not in {}'.format(
                          _value.tag, cls.__schema_classes__.keys())
                        LOGGER.error(_msg)<commit_msg>fix: make sure to not delete any objects that are not shared<commit_after>"""
Project: META-SHARE prototype implementation
 Author: <NAME> <<EMAIL>>
"""
import logging
import re

from django.core.cache import cache
from django.core.exceptions import ValidationError, ObjectDoesNotExist, \
  ImproperlyConfigured
from django.db.models.fields import related
from django.db import models, IntegrityError
from traceback import format_exc
from xml.etree.ElementTree import Element, fromstring, tostring

from metashare.repository.fields import MultiSelectField, MultiTextField, \
  MetaBooleanField, DictField

from metashare.settings import LOG_LEVEL, LOG_HANDLER, \
  CHECK_FOR_DUPLICATE_INSTANCES
from django.db.models.fields.related import ForeignRelatedObjectsDescriptor, \
    OneToOneField
from Queue import Queue
from metashare.storage.models import MASTER

# Setup logging support.
logging.basicConfig(level=LOG_LEVEL)
LOGGER = logging.getLogger('metashare.repository.supermodel')
LOGGER.addHandler(LOG_HANDLER)

# cfedermann: this prevents a bug with PostgreSQL databases and Django 1.3
try:
    from django.db.backends.postgresql_psycopg2.base import DatabaseFeatures
    DatabaseFeatures.can_return_id_from_insert = True

except ImproperlyConfigured:
    pass

REQUIRED = 1
OPTIONAL = 2
RECOMMENDED = 3

# template of a META-SHARE metadata XML schema URL
SCHEMA_URL = 'http://metashare.ilsp.gr/META-XMLSchema/v{0}/' \
  'META-SHARE-Resource.xsd'

METASHARE_ID_REGEXP = re.compile('<metashareId>.+</metashareId>',
  re.I|re.S|re.U)

OBJECT_XML_CACHE = {}

# pylint: disable-msg=W0611
from metashare import repository

def _remove_namespace_from_tags(element_tree):
    """
    Removes any namespace information from the given element_tree instance.
    """
    element_tree.tag = element_tree.tag.split('}')[-1]
    
    for child in element_tree.getchildren():
        _remove_namespace_from_tags(child)
    
    return element_tree

def _classify(class_name):
    """
    Converts the given class name into the corresponding class type object.
    """
    try:
        return eval('repository.models.{}'.format(class_name))

    except NameError:
        return None


def _make_choices_from_list(source_list):
    """
    Converts a given list of Strings to tuple choices.

    Returns a dictionary containing two keys:
    - max_length: the maximum char length for this source list,
    - choices: the list of (index, value) tuple choices.
    """
    _choices = []
    for index, value in enumerate(source_list):
        _choices.append((str(index), value))
    return {'max_length': len(_choices)/10+1, 'choices': tuple(_choices)}

def _make_choices_from_int_list(source_list):
    """
    Converts a given list of Integers to tuple choices.

    Returns a dictionary containing two keys:
    - max_length: the maximum char length for this source list,
    - choices: the list of (value, value) tuple choices.
    """
    _choices = []
    for value in source_list:
        _choices.append((value, value))
    return {'max_length': len(_choices)/10+1, 'choices': tuple(_choices)}


class SchemaModel(models.Model):
    """
    Super class for all XSD schema types/components.
    """
    __schema_name__ = 'UNDEFINED'
    __schema_fields__ = ()
    __schema_attrs__ = ()
    __schema_classes__ = {}
    __schema_parent__ = None
    
    class Meta:
        """
        This is an abstract super class for all schema models.
        """
        abstract = True

    @classmethod
    def is_required_field(cls, name):
        """
        Checks whether the field with the given name is a required field.
        """
        # pylint: disable-msg=E1101
        _fields = cls.get_fields()
        return name in _fields['required']

    @classmethod
    def get_many_to_many_fields(cls):
        """
        Returns a list containing all ManyToManyField fields for this model.
        """
        return [field.name for field in cls._meta.local_many_to_many]

    @classmethod
    def get_fields_flat(cls, unique_values=True):
        """
        Return all fields in a flat list
        """
        _fields = []
        for _, _field, _ in cls.__schema_fields__:
            if unique_values and _field in _fields:
                continue
            _fields.append(_field)

        return _fields

    @classmethod
    def get_fields(cls, unique_values=True):
        """
        Returns a dictionary containing all fields.
        """
        _fields = {'required': [], 'recommended': [], 'optional': []}
        for _not_used, _field, _required in cls.__schema_fields__:
            if _required == REQUIRED:
                if unique_values and _field in _fields['required']:
                    continue

                _fields['required'].append(_field)

            elif _required == RECOMMENDED:
                if unique_values and _field in _fields['recommended']:
                    continue

                _fields['recommended'].append(_field)

            else:
                if unique_values and _field in _fields['optional']:
                    continue

                _fields['optional'].append(_field)

        return _fields

    @classmethod
    def get_field_sets(cls, unique_values=True):
        """
        Returns a dictionary containing just "*_set" fields.
        """
        _field_sets = {'required': [], 'recommended': [], 'optional': []}
        for _not_used, _field, _required in cls.__schema_fields__:
            if _field.endswith('_set'):
                if _required == REQUIRED:
                    if unique_values and _field in _field_sets['required']:
                        continue

                    _field_sets['required'].append(_field)

                elif _required == RECOMMENDED:
                    if unique_values and _field in _field_sets['recommended']:
                        continue

                    _field_sets['recommended'].append(_field)

                else:
                    if unique_values and _field in _field_sets['optional']:
                        continue

                    _field_sets['optional'].append(_field)

        return _field_sets

    @classmethod
    def get_verbose_name(cls, fieldname):
        """
        For a real field such as 'firstName'
        or a pseudo-field such as 'contactinfotype_model_set',
        obtain the best possible verbose name that we can give.
        """
        #fields = cls._meta.get_field_by_name(fieldname)
        #if fields:
        #    field = fields[0]
        #    return field.verbose_name
        try:
            field = cls._meta.get_field(fieldname)
            return field.verbose_name
        except:
            remote = getattr(cls, fieldname, None)
            if isinstance(remote, ForeignRelatedObjectsDescriptor):
                return remote.related.model._meta.verbose_name
            return fieldname

    @staticmethod
    def _python_to_xml(value):
        """
        Converts the given Python value to its XML representation.
        """
        # Boolean values need to be rendered as 'true' or 'false' Strings.
        if isinstance(value, bool):
            if value:
                return 'true'
            else:
                return 'false'

        # String instances are converted to Unicode instances.
        elif isinstance(value, basestring):
            return unicode(value)

        # All other values are encoded as Strings.
        else:
            return str(value)

    @staticmethod
    def _xml_to_python(value, field):
        """
        Converts the given XML value to its Python representation for field.
        """
        # If no field is given, we have to return the value unmodified.
        if field is None:
            return value

        result = value

        # Handle fields with choices as we need to convert the current value
        # to the database representation value of the respective choice.
        if len(field.choices) > 0:
            # MetaBooleanField instances are a special case: here, we need to
            # return either 'Yes' or 'No', depending on the given value.
            if isinstance(field, MetaBooleanField):
                if value.strip().lower() in ('true', 'yes', '1'):
                    result = 'Yes'

                elif value.strip().lower() in ('false', 'no', '0'):
                    result = 'No'
                
                else:
                    LOGGER.error(u'Value {} not a valid MetaBooleanField ' \
                      'choice for {}'.format(repr(value), field.name))

            else:
                for _db_value, _readable_value in field.choices:
                    if _readable_value == value:
                        result = _db_value

                if result == value \
                        and not isinstance(field, models.IntegerField):
                    LOGGER.error(u'Value {} not found in choices for ' \
                      '{}'.format(repr(value), field.name))

        elif isinstance(field, models.BooleanField) \
          or isinstance(field, models.NullBooleanField):
            if value.strip().lower() == 'true':
                result = True

            elif value.strip().lower() == 'false':
                result = False

            else:
                LOGGER.error(u'Value {} not a valid Boolean for {}'.format(
                  repr(value), field.name))

        elif isinstance(field, models.TextField) \
          or isinstance(field, MultiTextField):
            if value is None:
                result = ''

        return result

    def export_to_elementtree(self):
        """
        Exports this instance to an XML ElementTree.
        """
        if self.__schema_name__ == "SUBCLASSABLE":
            # pylint: disable-msg=E1101
            return self.as_subclass().export_to_elementtree()

        _root = Element(self.__schema_name__)

        # Fix namespace attributes for the resourceInfo root element.
        if _root.tag == 'resourceInfo':
            # only import on demand (metashare.repository.models depends on
            # metashare.repository.supermodel, giving a circular dependency if
            # importin 'globally')
            from metashare.repository.models import SCHEMA_NAMESPACE, \
                SCHEMA_VERSION
            _root.attrib['xmlns'] = SCHEMA_NAMESPACE
            _root.attrib['xmlns:xsi'] = "http://www.w3.org/2001/" \
              "XMLSchema-instance"
            _root.attrib['xsi:schemaLocation'] = "{} {}" \
              .format(SCHEMA_NAMESPACE, SCHEMA_URL.format(SCHEMA_VERSION))

        # We first serialize all schema attributes.  For the moment, we just
        # assume that attributes are Strings only.  This holds for the v1.1
        # representative XSD, still more clever value handling might actually
        # be required for version v2; depends on the final schema...
        for _xsd_attr, _model_field, _not_used in self.__schema_attrs__:
            try:
                # First, we check if there is a get_FOO_display() method.
                _func = getattr(self, 'get_{0}_display'.format(_model_field))

                # If so, we call this method to determine the current value.
                _value = _func()

            except AttributeError:
                # Otherwise, we try to retrieve the value via getattr().
                _value = getattr(self, _model_field, None)

            # If a value could be found, it becomes an attribute of the node.
            if _value is not None:
                _root.attrib[_xsd_attr] = SchemaModel._python_to_xml(_value)

        #print self.get_fields()
        #print self.get_field_sets()

        # Then, we loop over all schema fields, retrieve their values and put
        # XML-ified versions of these values into the XML tree.
        for _xsd_field, _model_field, _not_used in self.__schema_fields__:
            try:
                # Again, we first check for a get_FOO_display() method.
                _func = getattr(self, 'get_{0}_display'.format(_model_field))
                _value = _func()

            except AttributeError:
                # Only falling back to getattr() if it doesn't exist.
                _value = getattr(self, _model_field, None)

            if _value is not None and _value != "":
                _model_set_value = _model_field.endswith('_model_set')

                if not _model_set_value:
                    _field = self._meta.get_field_by_name(_model_field)[0]
                else:
                    _field = None

                # For MetaBooleanFields, we actually need the database value
                # (i.e., not the displayed value).
                if isinstance(_field, MetaBooleanField):
                    _value = getattr(self, _model_field, [])

                # For MultiSelectFields, we call get_FOO_display_list().
                elif isinstance(_field, MultiSelectField):
                    _func = getattr(self, 'get_{0}_display_list'.format(
                      _model_field))
                    _value = _func()
                    
                    # Sort MultiSelectField values to allow comparison!
                    _value.sort()

                # For DictFields, we convert the dict to a list of tuples.
                elif isinstance(_field, DictField):
                    _value = _value.items()

                # For ManyToManyFields, compute all related objects.
                if isinstance(_value, models.Manager):
                    _value = _value.all().order_by('id')

                # If the value is not yet of list type, we wrap it in a list.
                elif not isinstance(_value, list):
                    _value = [_value]

                # Iterate over all values inside the current _value list.
                for _sub_value in _value:
                    # For SubclassableModel instances, we have to check if the
                    # sub class type matches the current XSD name; otherwise
                    # we have to ignore the current value.
                    if isinstance(_sub_value, SubclassableModel):
                        _clazz = _sub_value.as_subclass()
                        _clazz_name = _clazz.__class__.__name__
                        _choice_name = None
                        _xsd_name = _xsd_field.split('/')[-1]

                        if _xsd_name in self.__schema_classes__.keys():
                            _choice_name = self.__schema_classes__[_xsd_name]

                        if _clazz_name != _choice_name:
                            LOGGER.debug(u'Skipping choice value {}'.format(
                              _xsd_name))
                            continue

                        # If this is the correct sub class type, copy _clazz
                        # to the _sub_value variable for further processing...
                        _sub_value = _clazz

                    # Handle element paths inside _xsd_field names.  Supports
                    # element paths such as e.g. affiliation/OrganizationInfo.
                    _current_node = _root
                    _path = _xsd_field.split('/')
                    if len(_path) > 1:
                        # Iterate over all intermediate names and then create
                        # the corresponding element for these.
                        for _tag in _path[:-1]:
                            # Only create a new intermediate element if there
                            # is not already one available...
                            _element = None
                            if _model_set_value:
                                _element = _current_node.find(_tag)

                            if _element is None:
                                _element = Element(_tag)
                                _current_node.append(_element)

                            _current_node = _element

                    # The last name inside the path becomes our _xsd_name.
                    _xsd_name = _path[-1]

                    # For complex values, we recursively call this method.
                    if isinstance(_sub_value, SchemaModel):
                        if _sub_value.__schema_name__ == "STRINGMODEL":
                            _element = Element(_xsd_name)
                            _element.text = SchemaModel._python_to_xml(_sub_value.value)
                            _current_node.append(_element)

                        else:
                            _sub_value = _sub_value.export_to_elementtree()

                            # We fix the sub value's tag as it may be "wrong".
                            # E.g., PersonInfo is sometimes called contactPerson.
                            _sub_value.tag = _xsd_name

                            # And append the sub structure to the current node.
                            _current_node.append(_sub_value)

                    # Simple values are added to a new element and appended.
                    else:
                        _element = Element(_xsd_name)
                        if isinstance(_sub_value, tuple):
                            # tuple values come from DictFields with an RFC 3066
                            # language code key
                            _element.set('lang', _sub_value[0])
                            _element.text = SchemaModel._python_to_xml(
                                                                _sub_value[1])                            
                            if self.is_required_field(_model_field):
                                # If the element is "required" in the model,
                                # turn the "required" value of the element
                                # tree to "1" (true), otherwise "0" (false).
                                _element.required = self.is_required_field(
                                                                _model_field)
                            else:
                                _element.required = False
                        else:
                            _element.text = SchemaModel._python_to_xml(
                                                                _sub_value)                            
                            if self.is_required_field(_model_field):
                                _element.required = self.is_required_field(
                                                                _model_field)
                            else:
                                _element.required = False
                        _current_node.append(_element)

        # Return root node of the ElementTree; can be converted to String
        # using: xml.etree.ElementTree.tostring(_root, encoding="utf-8")
        return _root

    @classmethod
    def _check_for_duplicates(cls, _object):
        """
        Checks if the given _object is a redundant copy of another object.

        Returns a list containing all existing objects that are equal to the
        given _object instance, sorted by primary key 'id'.

        """
        if not CHECK_FOR_DUPLICATE_INSTANCES:
            return []

        _was_duplicate = False
        _related_objects = []

        # We collect all value constraints in a dictionary.
        kwargs = {}

        # Build list of field names for this class type.  Using introspection
        # with cls._meta.get_all_field_names() would not work as it contains
        # additional fields such as id and related fields that would break our
        # comparison code.
        _fields = [x[1] for x in cls.__schema_fields__]
        _fields.extend([x[1] for x in cls.__schema_attrs__])
        if hasattr(_object, 'copy_status'):
            _fields.append('copy_status')
        
        if _object.__schema_name__ == "STRINGMODEL":
            _fields.append("value")
        
        # Ensure that "back_to_" pointers are available for checking.
        for _field in cls._meta.fields:
            if isinstance(_field, related.ForeignKey):
                if _field.name.startswith('back_to_') and \
                  not _field.name in _fields:
                    _fields.append(_field.name)

        # We iterate over all model fields of the current _object.
        for field_name in _fields:
            # OneToMany fields need to be handled later, hence we add these to
            # our _related_objects list and continue with the loop.
            if field_name.endswith('_set'):
                _related_objects.append(field_name)
                LOGGER.debug(u'Skipping OneToMany "{}"'.format(field_name))
                continue

            # Retrieve the model field value using getattr().  This is enough
            # as even if a get_FOO_display() method would exist for a field,
            # the "raw" values needed to be compared anyway.
            _value = getattr(_object, field_name, None)

            # Retrieve model field instance to allow type checking.
            _field = _object._meta.get_field_by_name(field_name)[0]

            # For ManyToManyFields, compute the list of related objects.
            if isinstance(_field, related.ManyToManyField):
                _value = _value.all()

                # If there is no related object for this ManyToManyField, we
                # have to constrain kwargs[field_name] with the None value.
                if not len(_value):
                    _value = None

                # We have to use '__in' QuerySet lookup to avoid the "more
                # than one row returned by a subquery used as an expression"
                # bug which raises a DatabaseError otherwise.

                else:
                    field_name += '__in'

            # Finally, OneToOneField instances have to be checked as related
            # objects and hence need to be put into _related_objects.
            elif isinstance(_field, related.OneToOneField):
                LOGGER.debug(u'Skipping OneToOne "{}"'.format(field_name))
                _related_objects.append(field_name)
                continue

            # For ForeignKey instances, we have to check their name first.
            # "back_to_" fields need to be put into _related_objects, while
            # "normal" ForeignKey fields can be checked with our QuerySet.
            elif isinstance(_field, related.ForeignKey):
                if field_name.startswith('back_to_'):
                    LOGGER.debug(u'Skipping back_to "{}"'.format(field_name))
                    _related_objects.append(field_name)
                    continue

            # If the field value is not None or the field allows None as value
            # we set the current value as constraint in our kwargs dictionary.
            if _value is not None:
                kwargs[field_name] = _value

            elif _field.null:
                kwargs[field_name] = None

            # Otherwise, we print a message that we skip the current field.
            else:
                LOGGER.debug(u'Skipping {0}={1} ({2})'.format(field_name,
                  _value, type(_field)))

        # Use **magic to create a constrained QuerySet from kwargs.
        query_set = cls.objects.filter(**kwargs).order_by('id')

        _duplicates = []
        if query_set.count() > 1:
            # We now know that there may exist at least one duplicate for the
            # given _object;  we have to check the related objects to be sure.
            
            # Convert the current object into its serialised XML String and
            # remove any META-SHARE related id from this String.
            cache_key = '{}_{}'.format(type(_object).__name__.lower(),
              _object.id)
            if OBJECT_XML_CACHE.has_key(cache_key):
                _obj_value = OBJECT_XML_CACHE[cache_key]
            
            else:
                _obj_value = tostring(_object.export_to_elementtree())
                _obj_value = METASHARE_ID_REGEXP.sub('', _obj_value)
                OBJECT_XML_CACHE[cache_key] = _obj_value

            # Iterate over all potential duplicates, ordered by increasing id.
            for _candidate in query_set.iterator():
                # Skip the current candidate if it is our _object itself.
                if _candidate == _object:
                    continue

                # Convert candidate into XML String and remove META-SHARE id.
                cache_key = '{}_{}'.format(type(_candidate).__name__.lower(),
                  _candidate.id)
                if OBJECT_XML_CACHE.has_key(cache_key):
                    _check = OBJECT_XML_CACHE[cache_key]

                else:
                    _check = tostring(_candidate.export_to_elementtree())
                    _check = METASHARE_ID_REGEXP.sub('', _check)
                    OBJECT_XML_CACHE[cache_key] = _check

                # If both XML Strings are equal, we have found a duplicate!
                if _obj_value == _check:
                    _duplicates.append(_candidate)

        return _duplicates

    @staticmethod
    def _cleanup(objects, only_remove_duplicates=False):
        """
        Deletes all objects within the given objects list.

        The objects list contains tuples (obj, status) where obj is a Django
        object instance and status is in {'C', 'D', 'O'}.

        * 'C' == 'Created':   The corresponding object has just been created.

        * 'D' == 'Duplicate': The corresponding object is a duplicate.

        * 'O' == 'Original':  The corresponding object pre-existed in the DB.

        If only_remove_duplicates=True, only object instances with status='D'
        will be deleted while other instances are kept.
        
        Returns a list containing all instances which have not been deleted.

        """
        _objects = []
        for (obj, status) in objects:
            if only_remove_duplicates and status != 'D':
                LOGGER.debug(u'Keeping object {} ({})'.format(obj, status))
                _objects.append((obj, status))
                continue

            if obj.id:
                try:
                    LOGGER.debug(u'Deleting object {0}'.format(obj))
                    cache_key = '{}_{}'.format(type(obj).__name__.lower(),
                      obj.id)

                    if OBJECT_XML_CACHE.has_key(cache_key):
                        OBJECT_XML_CACHE.pop(cache_key)

                    if obj.__schema_name__ == "resourceInfo":
                        storage_object = obj.storage_object
                        storage_object.delete()
                    
                    obj.delete()             

                except ObjectDoesNotExist:
                    continue
        
        return _objects

    # pylint: disable-msg=R0911
    @classmethod
    def import_from_elementtree(
      cls, element_tree, cleanup=True, parent=None, copy_status=MASTER):
        """
        Imports the given XML ElementTree into an instance of type cls.

        Returns a tuple containing a reference to the created object instance
        as first value and the list of all objects that have been created when
        import the given XML ElementTree as second value.

        Returns (None, [], error_msg) in case of errors.
        """
        LOGGER.debug(u'parent: {0}'.format(parent))
        
        # We ignore name space information in tags, hence we remove it.
        element_tree = _remove_namespace_from_tags(element_tree)

        # First, we make sure that the given element_tree has the right tag.
        if element_tree is None \
          or element_tree.tag != cls.__schema_name__:
            _msg = u"Tags don't match: {}!={}".format(element_tree.tag,
              cls.__schema_name__)
            LOGGER.error(_msg)
            return (None, [], _msg)

        # We collect a list of all objects that are (recursively) created by
        # the importing process.  This is required if rollback has to be done.
        _created = []

        # We also need to instantiate a new instance of this class type.
        _object = cls()
        
        if hasattr(_object, 'copy_status'):
            _object.copy_status = copy_status

        # If a parent instance is given, add it to the object instance.
        if parent:
            _class_name = parent.__class__.__name__.lower()
            _foreign_key_name = 'back_to_{0}'.format(_class_name)

            try:
                LOGGER.debug(u'Setting schema parent {0}={1}'.format(
                  _foreign_key_name, parent))

                _ = _object._meta.get_field_by_name(_foreign_key_name)[0]
                setattr(_object, _foreign_key_name, parent)

            except (models.FieldDoesNotExist, AttributeError) as _exc:
                _msg = u"Could not set schema parent! ({})".format(_exc)
                LOGGER.error(_msg)
                SchemaModel._cleanup(_created)
                return (None, [], _msg)

        for xsd_attr, _model_field, _not_used in cls.__schema_attrs__:
            # Get attribute value and field instance for this xsd_attr.
            _attr = element_tree.attrib.get(xsd_attr, None)
            _field = _object._meta.get_field_by_name(_model_field)[0]

            # Convert attribute value to its Python representation for field.
            _value = SchemaModel._xml_to_python(_attr, _field)

            # If a value could be found, set the model field value.
            if _value is not None:
                setattr(_object, _model_field, _value)

        # Iterate over all model fields for an cls instance.  These need to be
        # sorted so that any _set fields are processed at the end!
        _fields_before_sets = list(cls.__schema_fields__)

        # The following allows to sort all ManyToMany field instances to the
        # end of the list; this is required to ensure that all required fields
        # can be handled and filled before saving this instance for handling
        # the ManyToMany fields...
        _fields_before_sets.sort(key=lambda x: x[2] == REQUIRED, reverse=True)

        _many_to_many_fields = []
        for _field in _fields_before_sets:
            if _field[1].endswith('_set'):
                continue

            _model_field = _object._meta.get_field_by_name(_field[1])[0]
            if isinstance(_model_field, related.ManyToManyField):
                _many_to_many_fields.append(_field)
                _fields_before_sets.remove(_field)

        _fields_before_sets.extend(_many_to_many_fields)

        _fields_before_sets.sort(cmp=lambda x, y: -y[1].endswith('_set'))

        for xsd_field, _model_field, _required in _fields_before_sets:
            # We want to collect all values for the current model field.  This
            # may be several, e.g., for multiple contactPerson elements.
            # Language tags currently only exist for DictFields. If the language
            # tags list is not empty, then its indexes will always correspond to
            # the indexes of the values list. 
            _values = []
            _lang_tags = []
            _parent = None

            # If we are handling a field set, we have to make sure that our
            # object instance has a valid identifier to be used as parent.
            # If the object instance's id is None, we have to save it first!
            if _model_field.endswith('_set'):
                if _object.id is None:
                    try:
                        LOGGER.debug(u'Saving parent object for {0}'.format(
                          _model_field))
                        _object.save()

                    except IntegrityError as _exc:
                        # pylint: disable-msg=E1101
                        _msg = u'Could not save {} object! ({})'.format(
                          _object.__class__, _exc)
                        LOGGER.error(_msg)

                        LOGGER.error(u'cls.__schema_parent__: {0}'.format(
                          cls.__schema_parent__))
                        LOGGER.error(u'parent: {0}'.format(parent))
                        if parent:
                            LOGGER.error(u'parent.__class__.__name__: ' \
                              u'{0}'.format(parent.__class__.__name__))

                        SchemaModel._cleanup(_created)
                        return (None, [], _msg)

                # Set current object instance as parent for recursive import.
                _parent = _object

            # Iterate over all sub elements with the current XSD field name.
            for _value in element_tree.findall(xsd_field):
                # As field sets are virtual, we set _field=None for them.
                if _model_field.endswith('_set'):
                    _field = None

                else:
                    _field = _object._meta.get_field_by_name(_model_field)[0]

                # The initial assumption is that duplicate objects should
                # be deleted immediately;  this does not hold for OneToOne
                # and OneToMany fields (where _parent is not None), only.
                _delete_duplicate_objects = True
                
                if isinstance(_field, related.OneToOneField) or _parent:
                    _delete_duplicate_objects = False

                # If the current value is simple-typed, append its text value.
                if not len(_value.getchildren()):
                    _text = SchemaModel._xml_to_python(_value.text, _field)
                    LOGGER.debug(u'_value.tag: {}, _value.text: {}'.format(
                      _value.tag, _text))

                    # We skip empty, simple-typed elements.
                    if not _text:
                        continue

                    # If the current field is a DictField, then we do not only
                    # have to fill the _values list
                    if isinstance(_field, DictField):
                        # 'und' is the ISO 639-2 special language code for an
                        # undetermined language
                        _lang_tags.append(_value.get('lang', 'und'))

                    elif _value.tag in cls.__schema_classes__.keys():
                        LOGGER.debug(u'_schema_classes__[{}] = {}'.format(
                          _value.tag, cls.__schema_classes__[_value.tag]))
                        _sub_cls = _classify(
                          cls.__schema_classes__[_value.tag])
                        if _sub_cls.__schema_name__ == "STRINGMODEL":
                            LOGGER.debug(u'Creating STRINGMODEL {}.'.format(
                              _text))
                            _instance = _sub_cls()
                            _instance.value = _text
                            _instance.save()

                            _duplicates = _sub_cls._check_for_duplicates(
                              _instance)
                            _was_duplicate = len(_duplicates) > 0

                            # Add current _instance instance to the _created
                            # list with correct status: 'D' if it was a
                            # duplicate, 'C' otherwise.
                            if _was_duplicate:
                                # If we are allowed to perform cleanup, we do
                                # so and also replace our _instance instance
                                # with the "original" object.
                                if _delete_duplicate_objects:
                                    SchemaModel._cleanup([(_instance, 'D')],
                                      only_remove_duplicates=True)

                                    # Replace _instance with "original".
                                    _instance = _duplicates[0]
                                    _created.append((_instance, 'O'))

                                else:
                                    _created.append((_instance, 'D'))

                            else:
                                _created.append((_instance, 'C'))

                            _text = _instance

                    _values.append(_text)

                # Otherwise, we have to handle the complex structure of value.
                else:
                    # For this, we have defined a mapping from XSD element
                    # names to Django model classes inside __schema_classes__.
                    if not _value.tag in cls.__schema_classes__.keys():
                        _msg = u'Unsupported tag: {} not in {}'.format(
                          _value.tag, cls.__schema_classes__.keys())
                        LOGGER.error(_msg)
{'fraction_non_alphanumeric': 0.07284207073744055, 'fraction_numerical': 0.002541955939430383, 'mean_word_length': 1.593132043376568, 'pattern_counts': {'":': 12, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 45, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1192478', 'n_tokens_mistral': 18873, 'n_tokens_neox': 17898, 'n_words': 5585}