<commit_before>import sys, os.path, operator, thread, threading
from operator import attrgetter
from itertools import count, izip, ifilterfalse

from pony import utils
from pony.thirdparty import etree

class OrmError(Exception): pass

class DiagramError(OrmError): pass
class SchemaError(OrmError): pass
class MappingError(OrmError): pass
class TransactionError(OrmError): pass
class ConstraintError(OrmError): pass
class CreateError(OrmError): pass
class UpdateError(OrmError): pass

DATA_HEADER = [ None, None ]

ROW_HEADER = [ None, None, 0, 0 ]
ROW_READ_MASK = 2
ROW_UPDATE_MASK = 3

class UnknownType(object):
    def __repr__(self): return 'UNKNOWN'

UNKNOWN = UnknownType()

next_id = count().next

class Attribute(object):
    def __init__(attr, py_type, *args, **keyargs):
        if attr.__class__ is Attribute:
            raise TypeError("'Atrribute' is abstract type")
        attr.pk_offset = None
        attr._id_ = next_id()
        attr.py_type = py_type
        attr.name = None
        attr.entity = None
        attr.args = args
        attr.options = keyargs
        try: attr.default = keyargs.pop('default')
        except KeyError: attr.default = None
        else:
            if attr.default is None and isinstance(attr, Required):
                raise TypeError(
                    'Default value for required attribute %s cannot be None'
                    % attr)
        attr.reverse = keyargs.pop('reverse', None)
        if attr.reverse is None: pass
        elif not isinstance(attr.reverse, (basestring, Attribute)):
            raise TypeError("Value of 'reverse' option must be name of "
                            "reverse attribute). Got: %r" % attr.reverse)
        elif not (attr.py_type, basestring):
            raise DiagramError('Reverse option cannot be set for this type %r'
                            % attr.py_type)
    def __str__(attr):
        owner_name = attr.entity is None and '?' or attr.entity.__name__
        return '%s.%s' % (owner_name, attr.name or '?')
    def __repr__(attr):
        return '<Attribute %s: %s>' % (attr, attr.__class__.__name__)
    def get_old(attr, obj):
        raise NotImplementedError
    def __get__(attr, obj, type=None):
        if obj is None: return attr
        try: return obj._pk_[attr.pk_offset]
        except TypeError: pass
        attr_info = obj._get_info().attrs[attr]
        trans = local.transaction
        data = trans.objects.get(obj)
        if data is None: raise NotImplementedError
        value = data[attr.new_offset]
        if value is UNKNOWN: raise NotImplementedError
        return value
    def __set__(attr, obj, value):
        pk = obj._pk_
        if attr.pk_offset is not None:
            if value == pk[attr.pk_offset]: return
            raise UpdateError('Cannot change value of primary key')
        if value is None and isinstance(attr, Required):
            raise UpdateError('Required attribute %s.%s cannot be set to None'
                              % (obj.__class__.__name__, attr.name))
        attr_info = obj._get_info().attrs[attr]
        trans = local.transaction
        data = trans.objects.get(obj)
        if data is None: raise NotImplementedError
        prev = data[attr.new_offset]
        if prev == value: return
        undo = []
        try:
            for key in obj._keys_:
                if key is obj._primary_key_: continue
                if attr not in key: continue
                position = list(key).index(attr)
                new_key = [ data[key_attr.new_offset] for key_attr in key ]
                old_key = tuple(new_key)
                new_key[position] = value
                if UNKNOWN in new_key: continue
                new_key = tuple(new_key)
                try: old_index, new_index = trans.indexes[key]
                except KeyError:
                    old_index, new_index = trans.indexes[key] = ({}, {})
                obj2 = new_index.setdefault(new_key, obj)
                if obj2 is not obj:
                    key_str = ', '.join(repr(item) for item in new_key)
                    raise UpdateError(
                        'Cannot update %s.%s: '
                        '%s with such unique index already exists: %s'
                        % (obj.__class__.__name__, attr.name,
                           obj2.__class__.__name__, key_str))
                if prev is not UNKNOWN:
                      del new_index[old_key]
                      undo.append((new_index, obj, old_key, new_key))
                else: undo.append((new_index, obj, None, new_key))
        except UpdateError:
            for new_index, obj, old_key, new_key in undo:
                del new_index[new_key]
                if old_key is not None: new_index[old_key] = obj
            raise
        if data[1] != 'C': data[1] = 'U'
        data[attr.new_offset] = value
        for table, column in attr_info.tables.items():
            cache = trans.caches.get(table)
            if cache is None: cache = trans.caches[table] = Cache(table)
            row = cache.rows.get(pk)
            if row is None: raise NotImplementedError # the cache remains in corrupted state
            assert row[0] is obj
            if row[1] != 'C':
                row[1] = 'U'
                row[ROW_UPDATE_MASK] |= column.mask
            row[column.new_offset] = value
    def __delete__(attr, obj):
        raise NotImplementedError

class Optional(Attribute):
    pass

class Required(Attribute):
    pass

class Unique(Required):
    def __new__(cls, *args, **keyargs):
        if not args: raise TypeError('Invalid count of positional arguments')
        attrs = tuple(a for a in args if isinstance(a, Attribute))
        non_attrs = [ a for a in args if not isinstance(a, Attribute) ]
        if attrs and non_attrs: raise TypeError('Invalid arguments')
        cls_dict = sys._getframe(1).f_locals
        keys = cls_dict.setdefault('_keys_', set())
        if issubclass(cls, PrimaryKey): tuple_class = _PrimaryKeyTuple
        else: tuple_class = tuple
        if not attrs:
            result = Required.__new__(cls, *args, **keyargs)
            keys.add(tuple_class((result,)))
            return result
        else: keys.add(tuple_class(attrs))

class PrimaryKey(Unique):
    pass

class _PrimaryKeyTuple(tuple):
    pass

class Collection(Attribute):
    pass

class Set(Collection):
    pass

##class List(Collection): pass
##class Dict(Collection): pass
##class Relation(Collection): pass

class Diagram(object):
    def __init__(diagram):
        diagram.lock = threading.RLock()
        diagram.entities = {} # entity_name -> Entity
        diagram.transactions = set()
    def clear(diagram):
        diagram.lock.acquire()
        try: diagram._clear()
        finally: diagram.lock.release()
    def _clear(diagram):
        diagram.lock.acquire()
        try:
            for trans in diagram.transactions: trans.data_source.clear_schema() # ????
        finally: diagram.lock.release()

class EntityMeta(type):
    def __init__(entity, name, bases, dict):
        super(EntityMeta, entity).__init__(name, bases, dict)
        if 'Entity' not in globals(): return
        outer_dict = sys._getframe(1).f_locals
        diagram = (dict.pop('_diagram_', None)
                   or outer_dict.get('_diagram_')
                   or outer_dict.setdefault('_diagram_', Diagram()))
        if not hasattr(diagram, 'data_source'):
            diagram.data_source = outer_dict.get('_data_source_')
        entity._cls_init_(diagram)
    def __setattr__(entity, name, value):
        entity._cls_setattr_(name, value)
    def __iter__(entity):
        return iter(())

class Entity(object):
    __metaclass__ = EntityMeta
    @classmethod
    def _cls_setattr_(entity, name, value):
        if name.startswith('_') and name.endswith('_'):
            type.__setattr__(entity, name, value)
        else: raise NotImplementedError
    @classmethod
    def _cls_init_(entity, diagram):
        if entity.__name__ in diagram.entities:
            raise DiagramError('Entity %s already exists' % entity.__name__)
        entity._objects_ = {}
        entity._lock_ = threading.Lock()
        direct_bases = [ c for c in entity.__bases__
                         if issubclass(c, Entity) and c is not Entity ]
        entity._direct_bases_ = direct_bases
        entity._all_bases_ = set((entity,))
        for base in direct_bases: entity._all_bases_.update(base._all_bases_)
        if direct_bases:
            roots = set(base._root_ for base in direct_bases)
            if len(roots) > 1: raise DiagramError(
                'With multiple inheritance of entities, '
                'inheritance graph must be diamond-like')
            entity._root_ = roots.pop()
            for base in direct_bases:
                if base._diagram_ is not diagram: raise DiagramError(
                    'When use inheritance, base and derived entities '
                    'must belong to same diagram')
        else: entity._root_ = entity

        base_attrs = []
        base_attrs_dict = {}
        for base in direct_bases:
            for a in base._attrs_:
                if base_attrs_dict.setdefault(a.name, a) is not a:
                    raise DiagramError('Ambiguous attribute name %s' % a.name)
                base_attrs.append(a)
        entity._base_attrs_ = base_attrs

        new_attrs = []
        for name, attr in entity.__dict__.items():
            if name in base_attrs_dict: raise DiagramError(
                'Name %s hide base attribute %s' % (name,base_attrs_dict[name]))
            if not isinstance(attr, Attribute): continue
            if name.startswith('_') and name.endswith('_'): raise DiagramError(
                'Attribute name cannot both starts and ends with underscore. '
                'Got: %s' % name)
            if attr.entity is not None:
                raise DiagramError('Duplicate use of attribute %s' % value)
            attr.name = name
            attr.entity = entity
            new_attrs.append(attr)
        new_attrs.sort(key=attrgetter('_id_'))
        entity._new_attrs_ = new_attrs

        entity._keys_ = keys = entity.__dict__.get('_keys_', set())
        primary_keys = set(key for key in keys
                               if isinstance(key, _PrimaryKeyTuple))
        if direct_bases:
            if primary_keys: raise DiagramError(
                'Primary key cannot be redefined in derived classes')
            for base in direct_bases: keys.update(base._keys_)
            primary_keys = set(key for key in keys
                                   if isinstance(key, _PrimaryKeyTuple))
                                   
        if len(primary_keys) > 1: raise DiagramError(
            'Only one primary key can be defined in each entity class')
        elif not primary_keys:
            if hasattr(entity, 'id'):
                raise DiagramError("Name 'id' alredy in use")
            _keys_ = set()
            attr = PrimaryKey(int) # Side effect: modifies _keys_ local variable
            attr.name = 'id'
            attr.entity = entity
            type.__setattr__(entity, 'id', attr)  # entity.id = attr
            entity._new_attrs_.insert(0, attr)
            key = _keys_.pop()
            entity._keys_.add(key)
            entity._primary_key_ = key
        else: entity._primary_key_ = primary_keys.pop()
        for i, attr in enumerate(entity._primary_key_): attr.pk_offset = i
        entity._pk_names_ = tuple(attr.name for attr in entity._primary_key_) # ???

        entity._attrs_ = base_attrs + new_attrs
        entity._attr_dict_ = dict((attr.name, attr) for attr in entity._attrs_)

        next_offset = count(len(DATA_HEADER)).next
        for attr in entity._attrs_:
            if attr.pk_offset is None:
                attr.old_offset = next_offset()
                attr.new_offset = next_offset()
            else: attr.old_offset = attr.new_offset = next_offset()
        data_size = entity._attrs_[-1].new_offset + 1
        entity._data_template_ = \
            DATA_HEADER + [ UNKNOWN ]*(data_size - len(DATA_HEADER))

        diagram.lock.acquire()
        try:
            diagram._clear()
            entity._diagram_ = diagram
            diagram.entities[entity.__name__] = entity
            entity._link_reverse_attrs_()
        finally: diagram.lock.release()

    @classmethod
    def _link_reverse_attrs_(entity):
        diagram = entity._diagram_
        for attr in entity._new_attrs_:
            py_type = attr.py_type
            if isinstance(py_type, basestring):
                entity2 = diagram.entities.get(py_type)
                if entity2 is None: continue
                attr.py_type = entity2
            elif issubclass(py_type, Entity):
                entity2 = py_type
                if entity2._diagram_ is not diagram: raise DiagramError(
                    'Interrelated entities must belong to same diagram. '
                    'Entities %s and %s belongs to different diagrams'
                    % (entity.__name__, entity2.__name__))
            else: continue
            
            reverse = attr.reverse
            if isinstance(reverse, basestring):
                attr2 = getattr(entity2, reverse, None)
                if attr2 is None: raise DiagramError(
                    'Reverse attribute %s.%s not found'
                    % (entity2.__name__, reverse))
            elif isinstance(reverse, Attribute):
                attr2 = reverse
                if attr2.entity is not entity2: raise DiagramError(
                    'Incorrect reverse attribute %s used in %s' % (attr2, attr))
            elif reverse is not None: raise DiagramError(
                "Value of 'reverse' option must be string. Got: %r"
                % type(reverse))
            else:
                candidates1 = []
                candidates2 = []
                for attr2 in entity2._new_attrs_:
                    if attr2.py_type not in (entity, entity.__name__): continue
                    reverse2 = attr2.reverse
                    if reverse2 in (attr, attr.name): candidates1.append(attr2)
                    elif reverse2 is None: candidates2.append(attr2)
                msg = 'Ambiguous reverse attribute for %s'
                if len(candidates1) > 1: raise DiagramError(msg % attr)
                elif len(candidates1) == 1: attr2 = candidates1[0]
                elif len(candidates2) > 1: raise DiagramError(msg % attr)
                elif len(candidates2) == 1: attr2 = candidates2[0]
                else: raise DiagramError(
                    'Reverse attribute for %s not found' % attr)

            type2 = attr2.py_type
            msg = 'Inconsistent reverse attributes %s and %s'
            if isinstance(type2, basestring):
                if type2 != entity.__name__:
                    raise DiagramError(msg % (attr, attr2))
                attr2.py_type = entity
            elif type2 != entity: raise DiagramError(msg % (attr, attr2))
            reverse2 = attr2.reverse
            if reverse2 not in (None, attr, attr.name):
                raise DiagramError(msg % (attr,attr2))

            attr.reverse = attr2
            attr2.reverse = attr
    @classmethod
    def _get_info(entity):
        trans = local.transaction
        if trans is None:
            data_source = entity._diagram_.data_source
            if data_source is None:
                outer_dict = sys._getframe(1).f_locals
                data_source = outer_dict.get('_data_source_')
            if data_source is not None: trans = Transaction(data_source)
            else: raise TransactionError(
                'There are no active transaction in thread %s. '
                'Cannot start transaction automatically, '
                'because default data source does not set'
                % thread.get_ident())
        else: data_source = trans.data_source
        info = data_source.entities.get(entity)
        if info is not None: return info
        data_source.generate_schema(entity._diagram_)
        return data_source.entities[entity]
    def __init__(obj, *args, **keyargs):
        raise TypeError('You cannot create entity instances directly. '
                        'Use Entity.create(...) or Entity.find(...) instead')
    @property
    def old(obj):
        return OldProxy(obj)
    @classmethod
    def create(entity, *args, **keyargs):
        if args:
            if len(args) != len(entity._primary_key_):
                raise CreateError('Invalid count of attrs in primary key')
            for name, value in zip(entity._pk_names_, args):
                if keyargs.setdefault(name, value) != value:
                    raise CreateError('Ambiguous attribute value for %r' % name)
        for name in ifilterfalse(entity._attr_dict_.__contains__, keyargs):
            raise CreateError('Unknown attribute %r' % name)
        pk = args or tuple(map(keyargs.get, entity._pk_names_))
        if None in pk: raise CreateError('Primary key is not specified')

        entity._lock_.acquire()
        try:
            obj = entity._objects_.get(pk)
            if obj is None:
                obj = object.__new__(entity)
                obj._pk_ = pk
                entity._objects_[pk] = obj
        finally: entity._lock_.release()

        data = entity._data_template_[:]
        data[0] = obj
        data[1] = 'C'
        for attr in entity._attrs_:
            try: value = keyargs[attr.name]
            except KeyError:
                value = attr.default
                msg = 'Required attribute %s.%s does not specified'
            else: msg = 'Value of required attribute %s.%s cannot be None'
            if value is None and isinstance(attr, Required):
                raise CreateError(msg % (entity.__name__, attr.name))
            data[attr.new_offset] = value

        info = entity._get_info()
        trans = local.transaction
        try:
            for key in entity._keys_:
                key_value = tuple(data[attr.new_offset] for attr in key)
                try: old_index, new_index = trans.indexes[key]
                except KeyError:
                    old_index, new_index = trans.indexes[key] = ({}, {})
                obj2 = new_index.setdefault(key_value, obj)
                if obj2 is not obj:
                    key_str = ', '.join(repr(item) for item in key_value)
                    if key is entity._primary_key_: key_type = 'primary key'
                    else: key_type = 'unique index'
                    raise CreateError(
                        '%s with such %s already exists: %s'
                        % (obj2.__class__.__name__, key_type, key_str))
        except CreateError, e:
            for key in entity._keys_:
                key_value = tuple(data[attr.new_offset] for attr in key)
                index_pair = trans.indexes.get(key)
                if index_pair is None: continue
                old_index, new_index = index_pair
                if new_index.get(key_value) is obj: del new_index[key_value]
            raise
        if trans.objects.setdefault(obj, data) is not data: raise AssertionError
        for table in info.tables:
            cache = trans.caches.get(table)
            if cache is None: cache = trans.caches[table] = Cache(table)
            new_row = cache.row_template[:]
            new_row[0] = obj
            new_row[1] = 'C'
            for column in table.columns:
                for attr in column.attrs:
                    if entity is attr.entity or issubclass(entity, attr.entity):
                        new_row[column.new_offset] = data[attr.new_offset]
                        break
                else: new_row[column.new_offset] = None
            if cache.rows.setdefault(pk, new_row) is not new_row:
                raise AssertionError
        return obj
    @classmethod
    def find(entity, *args, **keyargs):
        raise NotImplementedError
    def set(obj, **keyargs):
        info = obj._get_info()
        trans = local.transaction
        data = trans.objects.get(obj)
        if data is None: raise NotImplementedError
        old_data = data[:]
        attrs = set()
        for name, value in keyargs.items():
            attr = obj._attr_dict_.get(name)
            if attr is None: raise UpdateError("Unknown attribute: %r" % name)
            if data[attr.new_offset] == value: continue
            if attr.pk_offset is not None:
                raise UpdateError('Cannot change value of primary key')
            if value is None and isinstance(attr, Required):
                raise UpdateError(
                    'Required attribute %s.%s cannot be set to None'
                    % (obj.__class__.__name__, attr.name))
            attrs.add(attr)
            data[attr.new_offset] = value
        if not attrs: return
        undo = []
        try:
            for key in obj._keys_:
                if key is obj._primary_key_: continue
                new_key = tuple(data[attr.new_offset] for attr in key)
                if UNKNOWN in new_key: continue
                old_key = tuple(old_data[attr.new_offset] for attr in key)
                if old_key == new_key: continue
                try: old_index, new_index = trans.indexes[key]
                except KeyError:
                    old_index, new_index = trans.indexes[key] = ({}, {})
                obj2 = new_index.setdefault(new_key, obj)
                if obj2 is not obj:
                    key_str = ', '.join(repr(item) for item in new_key)
                    raise UpdateError(
                        'Cannot update %s.%s: '
                        '%s with such unique index already exists: %s'
                        % (obj.__class__.__name__, attr.name,
                           obj2.__class__.__name__, key_str))
                if UNKNOWN not in old_key:
                      del new_index[old_key]
                      undo.append((new_index, obj, old_key, new_key))
                else: undo.append((new_index, obj, None, new_key))
        except UpdateError:
            data[:] = old_data
            for new_index, obj, old_key, new_key in undo:
                del new_index[new_key]
                if old_key is not None: new_index[old_key] = obj
            raise
        if data[1] != 'C': data[1] = 'U'
        pk = obj._pk_
        for table in info.tables:
            cache = trans.caches.get(table)
            if cache is None: cache = trans.caches[table] = Cache(table)
            row = cache.rows.get(pk)
            if row is None: raise NotImplementedError # the cache remains in corrupted state
            assert row[0] is obj
            for attr in attrs:
                attr_info = info.attrs[attr]
                column = attr_info.tables.get(table)
                if column is None: continue
                if row[1] != 'C':
                    row[1] = 'U'
                    row[ROW_UPDATE_MASK] |= column.mask
                row[column.new_offset] = data[attr.new_offset]
        
def old(obj):
    return OldProxy(obj)

class OldProxy(object):
    def __init__(old_proxy, obj):
        cls = obj.__class__
        if not issubclass(cls, Entity): raise TypeError(<commit_msg>Fixed bug: attr.new_offset -> entity._new_offsets_[attr]<commit_after>import sys, os.path, operator, thread, threading
from operator import attrgetter
from itertools import count, izip, ifilterfalse

from pony import utils
from pony.thirdparty import etree

class OrmError(Exception): pass

class DiagramError(OrmError): pass
class SchemaError(OrmError): pass
class MappingError(OrmError): pass
class TransactionError(OrmError): pass
class ConstraintError(OrmError): pass
class CreateError(OrmError): pass
class UpdateError(OrmError): pass

DATA_HEADER = [ None, None ]

ROW_HEADER = [ None, None, 0, 0 ]
ROW_READ_MASK = 2
ROW_UPDATE_MASK = 3

class UnknownType(object):
    def __repr__(self): return 'UNKNOWN'

UNKNOWN = UnknownType()

next_id = count().next

class Attribute(object):
    def __init__(attr, py_type, *args, **keyargs):
        if attr.__class__ is Attribute:
            raise TypeError("'Atrribute' is abstract type")
        attr.pk_offset = None
        attr._id_ = next_id()
        attr.py_type = py_type
        attr.name = None
        attr.entity = None
        attr.args = args
        attr.options = keyargs
        try: attr.default = keyargs.pop('default')
        except KeyError: attr.default = None
        else:
            if attr.default is None and isinstance(attr, Required):
                raise TypeError(
                    'Default value for required attribute %s cannot be None'
                    % attr)
        attr.reverse = keyargs.pop('reverse', None)
        if attr.reverse is None: pass
        elif not isinstance(attr.reverse, (basestring, Attribute)):
            raise TypeError("Value of 'reverse' option must be name of "
                            "reverse attribute). Got: %r" % attr.reverse)
        elif not (attr.py_type, basestring):
            raise DiagramError('Reverse option cannot be set for this type %r'
                            % attr.py_type)
    def __str__(attr):
        owner_name = attr.entity is None and '?' or attr.entity.__name__
        return '%s.%s' % (owner_name, attr.name or '?')
    def __repr__(attr):
        return '<Attribute %s: %s>' % (attr, attr.__class__.__name__)
    def get_old(attr, obj):
        raise NotImplementedError
    def __get__(attr, obj, type=None):
        if obj is None: return attr
        try: return obj._pk_[attr.pk_offset]
        except TypeError: pass
        attr_info = obj._get_info().attrs[attr]
        trans = local.transaction
        data = trans.objects.get(obj)
        if data is None: raise NotImplementedError
        value = data[obj._new_offsets_[attr]]
        if value is UNKNOWN: raise NotImplementedError
        return value
    def __set__(attr, obj, value):
        pk = obj._pk_
        if attr.pk_offset is not None:
            if value == pk[attr.pk_offset]: return
            raise UpdateError('Cannot change value of primary key')
        if value is None and isinstance(attr, Required):
            raise UpdateError('Required attribute %s.%s cannot be set to None'
                              % (obj.__class__.__name__, attr.name))
        attr_info = obj._get_info().attrs[attr]
        trans = local.transaction
        data = trans.objects.get(obj)
        if data is None: raise NotImplementedError
        prev = data[obj._new_offsets_[attr]]
        if prev == value: return
        undo = []
        try:
            for key in obj._keys_:
                if key is obj._primary_key_: continue
                if attr not in key: continue
                position = list(key).index(attr)
                new_key = map(data.__getitem__,
                              map(obj._new_offsets_.__getitem__, key))
                old_key = tuple(new_key)
                new_key[position] = value
                if UNKNOWN in new_key: continue
                new_key = tuple(new_key)
                try: old_index, new_index = trans.indexes[key]
                except KeyError:
                    old_index, new_index = trans.indexes[key] = ({}, {})
                obj2 = new_index.setdefault(new_key, obj)
                if obj2 is not obj:
                    key_str = ', '.join(repr(item) for item in new_key)
                    raise UpdateError(
                        'Cannot update %s.%s: '
                        '%s with such unique index already exists: %s'
                        % (obj.__class__.__name__, attr.name,
                           obj2.__class__.__name__, key_str))
                if prev is not UNKNOWN:
                      del new_index[old_key]
                      undo.append((new_index, obj, old_key, new_key))
                else: undo.append((new_index, obj, None, new_key))
        except UpdateError:
            for new_index, obj, old_key, new_key in undo:
                del new_index[new_key]
                if old_key is not None: new_index[old_key] = obj
            raise
        if data[1] != 'C': data[1] = 'U'
        data[obj._new_offsets_[attr]] = value
        for table, column in attr_info.tables.items():
            cache = trans.caches.get(table)
            if cache is None: cache = trans.caches[table] = Cache(table)
            row = cache.rows.get(pk)
            if row is None: raise NotImplementedError # the cache remains in corrupted state
            assert row[0] is obj
            if row[1] != 'C':
                row[1] = 'U'
                row[ROW_UPDATE_MASK] |= column.mask
            row[column.new_offset] = value
    def __delete__(attr, obj):
        raise NotImplementedError

class Optional(Attribute):
    pass

class Required(Attribute):
    pass

class Unique(Required):
    def __new__(cls, *args, **keyargs):
        if not args: raise TypeError('Invalid count of positional arguments')
        attrs = tuple(a for a in args if isinstance(a, Attribute))
        non_attrs = [ a for a in args if not isinstance(a, Attribute) ]
        if attrs and non_attrs: raise TypeError('Invalid arguments')
        cls_dict = sys._getframe(1).f_locals
        keys = cls_dict.setdefault('_keys_', set())
        if issubclass(cls, PrimaryKey): tuple_class = _PrimaryKeyTuple
        else: tuple_class = tuple
        if not attrs:
            result = Required.__new__(cls, *args, **keyargs)
            keys.add(tuple_class((result,)))
            return result
        else: keys.add(tuple_class(attrs))

class PrimaryKey(Unique):
    pass

class _PrimaryKeyTuple(tuple):
    pass

class Collection(Attribute):
    pass

class Set(Collection):
    pass

##class List(Collection): pass
##class Dict(Collection): pass
##class Relation(Collection): pass

class Diagram(object):
    def __init__(diagram):
        diagram.lock = threading.RLock()
        diagram.entities = {} # entity_name -> Entity
        diagram.transactions = set()
    def clear(diagram):
        diagram.lock.acquire()
        try: diagram._clear()
        finally: diagram.lock.release()
    def _clear(diagram):
        diagram.lock.acquire()
        try:
            for trans in diagram.transactions: trans.data_source.clear_schema() # ????
        finally: diagram.lock.release()

class EntityMeta(type):
    def __init__(entity, name, bases, dict):
        super(EntityMeta, entity).__init__(name, bases, dict)
        if 'Entity' not in globals(): return
        outer_dict = sys._getframe(1).f_locals
        diagram = (dict.pop('_diagram_', None)
                   or outer_dict.get('_diagram_')
                   or outer_dict.setdefault('_diagram_', Diagram()))
        if not hasattr(diagram, 'data_source'):
            diagram.data_source = outer_dict.get('_data_source_')
        entity._cls_init_(diagram)
    def __setattr__(entity, name, value):
        entity._cls_setattr_(name, value)
    def __iter__(entity):
        return iter(())

class Entity(object):
    __metaclass__ = EntityMeta
    @classmethod
    def _cls_setattr_(entity, name, value):
        if name.startswith('_') and name.endswith('_'):
            type.__setattr__(entity, name, value)
        else: raise NotImplementedError
    @classmethod
    def _cls_init_(entity, diagram):
        if entity.__name__ in diagram.entities:
            raise DiagramError('Entity %s already exists' % entity.__name__)
        entity._objects_ = {}
        entity._lock_ = threading.Lock()
        direct_bases = [ c for c in entity.__bases__
                         if issubclass(c, Entity) and c is not Entity ]
        entity._direct_bases_ = direct_bases
        entity._all_bases_ = set((entity,))
        for base in direct_bases: entity._all_bases_.update(base._all_bases_)
        if direct_bases:
            roots = set(base._root_ for base in direct_bases)
            if len(roots) > 1: raise DiagramError(
                'With multiple inheritance of entities, '
                'inheritance graph must be diamond-like')
            entity._root_ = roots.pop()
            for base in direct_bases:
                if base._diagram_ is not diagram: raise DiagramError(
                    'When use inheritance, base and derived entities '
                    'must belong to same diagram')
        else: entity._root_ = entity

        base_attrs = []
        base_attrs_dict = {}
        for base in direct_bases:
            for a in base._attrs_:
                if base_attrs_dict.setdefault(a.name, a) is not a:
                    raise DiagramError('Ambiguous attribute name %s' % a.name)
                base_attrs.append(a)
        entity._base_attrs_ = base_attrs

        new_attrs = []
        for name, attr in entity.__dict__.items():
            if name in base_attrs_dict: raise DiagramError(
                'Name %s hide base attribute %s' % (name,base_attrs_dict[name]))
            if not isinstance(attr, Attribute): continue
            if name.startswith('_') and name.endswith('_'): raise DiagramError(
                'Attribute name cannot both starts and ends with underscore. '
                'Got: %s' % name)
            if attr.entity is not None:
                raise DiagramError('Duplicate use of attribute %s' % value)
            attr.name = name
            attr.entity = entity
            new_attrs.append(attr)
        new_attrs.sort(key=attrgetter('_id_'))
        entity._new_attrs_ = new_attrs

        entity._keys_ = keys = entity.__dict__.get('_keys_', set())
        primary_keys = set(key for key in keys
                               if isinstance(key, _PrimaryKeyTuple))
        if direct_bases:
            if primary_keys: raise DiagramError(
                'Primary key cannot be redefined in derived classes')
            for base in direct_bases: keys.update(base._keys_)
            primary_keys = set(key for key in keys
                                   if isinstance(key, _PrimaryKeyTuple))
                                   
        if len(primary_keys) > 1: raise DiagramError(
            'Only one primary key can be defined in each entity class')
        elif not primary_keys:
            if hasattr(entity, 'id'):
                raise DiagramError("Name 'id' alredy in use")
            _keys_ = set()
            attr = PrimaryKey(int) # Side effect: modifies _keys_ local variable
            attr.name = 'id'
            attr.entity = entity
            type.__setattr__(entity, 'id', attr)  # entity.id = attr
            entity._new_attrs_.insert(0, attr)
            key = _keys_.pop()
            entity._keys_.add(key)
            entity._primary_key_ = key
        else: entity._primary_key_ = primary_keys.pop()
        for i, attr in enumerate(entity._primary_key_): attr.pk_offset = i
        entity._pk_names_ = tuple(attr.name for attr in entity._primary_key_) # ???

        entity._attrs_ = base_attrs + new_attrs
        entity._attr_dict_ = dict((attr.name, attr) for attr in entity._attrs_)

        next_offset = count(len(DATA_HEADER)).next
        entity._old_offsets_ = old_offsets = {}
        entity._new_offsets_ = new_offsets = {}
        for attr in entity._attrs_:
            if attr.pk_offset is None:
                old_offsets[attr] = next_offset()
                new_offsets[attr] = next_offset()
            else: old_offsets[attr] = new_offsets[attr] = next_offset()
        data_size = next_offset()
        entity._data_template_ = \
            DATA_HEADER + [ UNKNOWN ]*(data_size - len(DATA_HEADER))

        diagram.lock.acquire()
        try:
            diagram._clear()
            entity._diagram_ = diagram
            diagram.entities[entity.__name__] = entity
            entity._link_reverse_attrs_()
        finally: diagram.lock.release()

    @classmethod
    def _link_reverse_attrs_(entity):
        diagram = entity._diagram_
        for attr in entity._new_attrs_:
            py_type = attr.py_type
            if isinstance(py_type, basestring):
                entity2 = diagram.entities.get(py_type)
                if entity2 is None: continue
                attr.py_type = entity2
            elif issubclass(py_type, Entity):
                entity2 = py_type
                if entity2._diagram_ is not diagram: raise DiagramError(
                    'Interrelated entities must belong to same diagram. '
                    'Entities %s and %s belongs to different diagrams'
                    % (entity.__name__, entity2.__name__))
            else: continue
            
            reverse = attr.reverse
            if isinstance(reverse, basestring):
                attr2 = getattr(entity2, reverse, None)
                if attr2 is None: raise DiagramError(
                    'Reverse attribute %s.%s not found'
                    % (entity2.__name__, reverse))
            elif isinstance(reverse, Attribute):
                attr2 = reverse
                if attr2.entity is not entity2: raise DiagramError(
                    'Incorrect reverse attribute %s used in %s' % (attr2, attr))
            elif reverse is not None: raise DiagramError(
                "Value of 'reverse' option must be string. Got: %r"
                % type(reverse))
            else:
                candidates1 = []
                candidates2 = []
                for attr2 in entity2._new_attrs_:
                    if attr2.py_type not in (entity, entity.__name__): continue
                    reverse2 = attr2.reverse
                    if reverse2 in (attr, attr.name): candidates1.append(attr2)
                    elif reverse2 is None: candidates2.append(attr2)
                msg = 'Ambiguous reverse attribute for %s'
                if len(candidates1) > 1: raise DiagramError(msg % attr)
                elif len(candidates1) == 1: attr2 = candidates1[0]
                elif len(candidates2) > 1: raise DiagramError(msg % attr)
                elif len(candidates2) == 1: attr2 = candidates2[0]
                else: raise DiagramError(
                    'Reverse attribute for %s not found' % attr)

            type2 = attr2.py_type
            msg = 'Inconsistent reverse attributes %s and %s'
            if isinstance(type2, basestring):
                if type2 != entity.__name__:
                    raise DiagramError(msg % (attr, attr2))
                attr2.py_type = entity
            elif type2 != entity: raise DiagramError(msg % (attr, attr2))
            reverse2 = attr2.reverse
            if reverse2 not in (None, attr, attr.name):
                raise DiagramError(msg % (attr,attr2))

            attr.reverse = attr2
            attr2.reverse = attr
    @classmethod
    def _get_info(entity):
        trans = local.transaction
        if trans is None:
            data_source = entity._diagram_.data_source
            if data_source is None:
                outer_dict = sys._getframe(1).f_locals
                data_source = outer_dict.get('_data_source_')
            if data_source is not None: trans = Transaction(data_source)
            else: raise TransactionError(
                'There are no active transaction in thread %s. '
                'Cannot start transaction automatically, '
                'because default data source does not set'
                % thread.get_ident())
        else: data_source = trans.data_source
        info = data_source.entities.get(entity)
        if info is not None: return info
        data_source.generate_schema(entity._diagram_)
        return data_source.entities[entity]
    def __init__(obj, *args, **keyargs):
        raise TypeError('You cannot create entity instances directly. '
                        'Use Entity.create(...) or Entity.find(...) instead')
    @property
    def old(obj):
        return OldProxy(obj)
    @classmethod
    def create(entity, *args, **keyargs):
        if args:
            if len(args) != len(entity._primary_key_):
                raise CreateError('Invalid count of attrs in primary key')
            for name, value in zip(entity._pk_names_, args):
                if keyargs.setdefault(name, value) != value:
                    raise CreateError('Ambiguous attribute value for %r' % name)
        for name in ifilterfalse(entity._attr_dict_.__contains__, keyargs):
            raise CreateError('Unknown attribute %r' % name)
        pk = args or tuple(map(keyargs.get, entity._pk_names_))
        if None in pk: raise CreateError('Primary key is not specified')

        entity._lock_.acquire()
        try:
            obj = entity._objects_.get(pk)
            if obj is None:
                obj = object.__new__(entity)
                obj._pk_ = pk
                entity._objects_[pk] = obj
        finally: entity._lock_.release()

        data = entity._data_template_[:]
        data[0] = obj
        data[1] = 'C'
        for attr in entity._attrs_:
            try: value = keyargs[attr.name]
            except KeyError:
                value = attr.default
                msg = 'Required attribute %s.%s does not specified'
            else: msg = 'Value of required attribute %s.%s cannot be None'
            if value is None and isinstance(attr, Required):
                raise CreateError(msg % (entity.__name__, attr.name))
            data[entity._new_offsets_[attr]] = value

        info = entity._get_info()
        trans = local.transaction
        try:
            for key in entity._keys_:
                key_value = tuple(map(data.__getitem__,
                    map(entity._new_offsets_.__getitem__, key)))
                try: old_index, new_index = trans.indexes[key]
                except KeyError:
                    old_index, new_index = trans.indexes[key] = ({}, {})
                obj2 = new_index.setdefault(key_value, obj)
                if obj2 is not obj:
                    key_str = ', '.join(repr(item) for item in key_value)
                    if key is entity._primary_key_: key_type = 'primary key'
                    else: key_type = 'unique index'
                    raise CreateError(
                        '%s with such %s already exists: %s'
                        % (obj2.__class__.__name__, key_type, key_str))
        except CreateError, e:
            for key in entity._keys_:
                key_value = tuple(map(data.__getitem__,
                    map(entity._new_offsets_.__getitem__, key)))
                index_pair = trans.indexes.get(key)
                if index_pair is None: continue
                old_index, new_index = index_pair
                if new_index.get(key_value) is obj: del new_index[key_value]
            raise
        if trans.objects.setdefault(obj, data) is not data: raise AssertionError
        for table in info.tables:
            cache = trans.caches.get(table)
            if cache is None: cache = trans.caches[table] = Cache(table)
            new_row = cache.row_template[:]
            new_row[0] = obj
            new_row[1] = 'C'
            for column in table.columns:
                for attr in column.attrs:
                    if entity is attr.entity or issubclass(entity, attr.entity):
                        value = data[entity._new_offsets_[attr]]
                        new_row[column.new_offset] = value
                        break
                else: new_row[column.new_offset] = None
            if cache.rows.setdefault(pk, new_row) is not new_row:
                raise AssertionError
        return obj
    @classmethod
    def find(entity, *args, **keyargs):
        raise NotImplementedError
    def set(obj, **keyargs):
        info = obj._get_info()
        trans = local.transaction
        data = trans.objects.get(obj)
        if data is None: raise NotImplementedError
        old_data = data[:]
        attrs = set()
        for name, value in keyargs.items():
            attr = obj._attr_dict_.get(name)
            if attr is None: raise UpdateError("Unknown attribute: %r" % name)
            if data[entity._new_offsets_[attr]] == value: continue
            if attr.pk_offset is not None:
                raise UpdateError('Cannot change value of primary key')
            if value is None and isinstance(attr, Required):
                raise UpdateError(
                    'Required attribute %s.%s cannot be set to None'
                    % (obj.__class__.__name__, attr.name))
            attrs.add(attr)
            data[entity._new_offsets_[attr]] = value
        if not attrs: return
        undo = []
        try:
            for key in obj._keys_:
                if key is obj._primary_key_: continue
                new_key = tuple(map(data.__getitem__,
                    map(entity._new_offsets_.__getitem__, key)))
                if UNKNOWN in new_key: continue
                old_key = tuple(map(old_data.__getitem__,
                    map(entity._new_offsets_.__getitem__, key)))
                if old_key == new_key: continue
                try: old_index, new_index = trans.indexes[key]
                except KeyError:
                    old_index, new_index = trans.indexes[key] = ({}, {})
                obj2 = new_index.setdefault(new_key, obj)
                if obj2 is not obj:
                    key_str = ', '.join(repr(item) for item in new_key)
                    raise UpdateError(
                        'Cannot update %s.%s: '
                        '%s with such unique index already exists: %s'
                        % (obj.__class__.__name__, attr.name,
                           obj2.__class__.__name__, key_str))
                if UNKNOWN not in old_key:
                      del new_index[old_key]
                      undo.append((new_index, obj, old_key, new_key))
                else: undo.append((new_index, obj, None, new_key))
        except UpdateError:
            data[:] = old_data
            for new_index, obj, old_key, new_key in undo:
                del new_index[new_key]
                if old_key is not None: new_index[old_key] = obj
            raise
        if data[1] != 'C': data[1] = 'U'
        pk = obj._pk_
        for table in info.tables:
            cache = trans.caches.get(table)
            if cache is None: cache = trans.caches[table] = Cache(table)
            row = cache.rows.get(pk)
            if row is None: raise NotImplementedError # the cache remains in corrupted state
            assert row[0] is obj
            for attr in attrs:
                attr_info = info.attrs[attr]
                column = attr_info.tables.get(table)
                if column is None: continue
                if row[1] != 'C':
                    row[1] = 'U'
                    row[ROW_UPDATE_MASK] |= column.mask
                value = data[entity._new_offsets_[attr]]
                row[column.new_offset] = value
        
def old(obj):
    return OldProxy(obj)

class OldProxy(object):
    def __init__(old_proxy, obj):
        cls = obj.__class__
        if not issubclass(cls, Entity): raise TypeError(
{'fraction_non_alphanumeric': 0.08901881577548634, 'fraction_numerical': 0.003784415860529393, 'mean_word_length': 1.8139994017349685, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3651484', 'n_tokens_mistral': 13075, 'n_tokens_neox': 12658, 'n_words': 3673}