<commit_before>"""Model and Property classes and associated stuff.

TODO: docstrings, style, asserts
"""

import calendar
import datetime
import logging

from google.appengine.datastore import entity_pb

from core import datastore_rpc

# NOTE: Key is meant for export, too.
from ndb.key import Key, _ReferenceFromPairs, _DefaultAppId

kind_map = {}  # Dict mapping {kind: Model subclass}

class ModelAdapter(datastore_rpc.AbstractAdapter):

  def pb_to_key(self, pb):
    return Key(reference=pb)

  def key_to_pb(self, key):
    return key.reference()

  def pb_to_entity(self, pb):
    kind = None
    if pb.has_key():
      key = Key(reference=pb.key())  # TODO: Avoid doing this twice
      for kind, _ in key.pairs():
        pass  # As a side effect, set kind to the last kind, if any
    modelclass = kind_map.get(kind, Model)
    ent = modelclass()
    ent.FromPb(pb)
    return ent

  def entity_to_pb(self, ent):
    pb = ent.ToPb()
    return pb

conn = datastore_rpc.Connection(adapter=ModelAdapter())

class Model(object):
  """A mutable datastore entity."""

  # TODO: Prevent accidental attribute assignments

  # Class variables updated by FixUpProperties()
  _properties = None
  _has_repeated = False

  # Defaults for instance variables.
  _key = None
  _values = None

  # TODO: Make _ versions of all methods, and make non-_ versions
  # simple aliases. That way the _ version is still accessible even if
  # the non-_ version has been obscured by a property.

  # TODO: Distinguish between purposes: to call FromPb() or setvalue() etc.
  @datastore_rpc._positional(1)
  def __init__(self, key=None, **kwds):
    cls = self.__class__
    if cls is not Model and cls._properties is None:
      FixUpProperties(cls)
    self._key = key
    self._values = {}
    for name, value in kwds.iteritems():
      prop = getattr(cls, name)
      assert isinstance(prop, Property)
      prop.SetValue(self, value)

  def __repr__(self):
    args = []
    if self._key is not None:
      args.append('key=%r' % self._key)
    for name_value in sorted(self._values.iteritems()):
      args.append('%s=%r' % name_value)
    s = '%s(%s)' % (self.__class__.__name__, ', '.join(args))
    return s

  # TODO: Make kind a property also?
  @classmethod
  def getkind(cls):
    return cls.__name__

  def _getkey(self):
    return self._key

  def _setkey(self, key):
    if key is not None:
      assert isinstance(key, Key)
      if self.__class__ is not Model:
        assert list(key.pairs())[-1][0] == self.getkind()
    self._key = key

  def _delkey(self):
    self._key = None

  key = property(_getkey, _setkey, _delkey)

  def __hash__(self):
    raise TypeError('Model is not immutable')

  def __eq__(self, other):
    if other.__class__ is not self.__class__:
      return NotImplemented
    # It's okay to use private names -- we're the same class
    if self._key != other._key:
      # TODO: If one key is None and the other is an explicit
      # incomplete key of the simplest form, this should be OK.
      return False
    # Ignore differences in values that are None.
    self_values = [(name, value)
                   for name, value in self._values.iteritems()
                   if value is not None]
    other_values = [(name, value)
                    for name, value in other._values.iteritems()
                    if value is not None]
    return self_values == other_values

  def __ne__(self, other):
    eq = self.__eq__(other)
    if eq is  NotImplemented:
      return NotImplemented
    return not eq

  def ToPb(self):
    pb = entity_pb.EntityProto()

    # TODO: Move the key stuff into ModelAdapter.entity_to_pb()?
    key = self._key
    if key is None:
      ref = _ReferenceFromPairs([(self.getkind(), None)], pb.mutable_key())
      ref.set_app(_DefaultAppId())
    else:
      ref = key._Key__reference  # Don't copy
      pb.mutable_key().CopyFrom(ref)
    group = pb.mutable_entity_group()
    elem = ref.path().element(0)
    if elem.id() or elem.name():
      group.add_element().CopyFrom(elem)

    if self._properties:
      # TODO: Sort by property declaration order
      for name, prop in sorted(self._properties.iteritems()):
        prop.Serialize(self, pb)

    return pb

  # TODO: Make this a class method?
  def FromPb(self, pb):
    assert not self._key
    assert not self._values
    assert isinstance(pb, entity_pb.EntityProto)

    # TODO: Move the key stuff into ModelAdapter.pb_to_entity()?
    if pb.has_key():
      self._key = Key(reference=pb.key())

    indexed_properties = pb.property_list()
    unindexed_properties = pb.raw_property_list()
    for plist in [indexed_properties, unindexed_properties]:
      for p in plist:
        prop = self.GetPropertyFor(p, plist is indexed_properties)
        prop.Deserialize(self, p)

  def GetPropertyFor(self, p, indexed=True, depth=0):
    db_name = p.name()
    parts = db_name.split('.')
    if len(parts) <= depth:
      import pdb; pdb.set_trace()
    assert len(parts) > depth, (p.name(), parts, depth)
    next = parts[depth]
    prop = None
    if self._properties:
      prop = self._properties.get(next)
    if prop is None:
      prop = self.FakeProperty(p, next, indexed)
    return prop

  def FakeProperty(self, p, next, indexed=True):
    cls = self.__class__
    if self._properties is cls._properties:
      self._properties = dict(cls._properties or ())

    if p.name() != next and not p.name().endswith('.' + next):
      prop = StructuredProperty(Model, next)
      pid = str(id(prop))
      assert pid not in self._values
      self._values[pid] = Model()
    else:
      prop = GenericProperty(next,
                             repeated=p.multiple(),
                             indexed=indexed)

    prop.FixUp(str(id(prop)))  # Use a unique string as Python name.
<commit_msg>Kill pdb invocation.<commit_after>"""Model and Property classes and associated stuff.

TODO: docstrings, style, asserts
"""

import calendar
import datetime
import logging

from google.appengine.datastore import entity_pb

from core import datastore_rpc

# NOTE: Key is meant for export, too.
from ndb.key import Key, _ReferenceFromPairs, _DefaultAppId

kind_map = {}  # Dict mapping {kind: Model subclass}

class ModelAdapter(datastore_rpc.AbstractAdapter):

  def pb_to_key(self, pb):
    return Key(reference=pb)

  def key_to_pb(self, key):
    return key.reference()

  def pb_to_entity(self, pb):
    kind = None
    if pb.has_key():
      key = Key(reference=pb.key())  # TODO: Avoid doing this twice
      for kind, _ in key.pairs():
        pass  # As a side effect, set kind to the last kind, if any
    modelclass = kind_map.get(kind, Model)
    ent = modelclass()
    ent.FromPb(pb)
    return ent

  def entity_to_pb(self, ent):
    pb = ent.ToPb()
    return pb

conn = datastore_rpc.Connection(adapter=ModelAdapter())

class Model(object):
  """A mutable datastore entity."""

  # TODO: Prevent accidental attribute assignments

  # Class variables updated by FixUpProperties()
  _properties = None
  _has_repeated = False

  # Defaults for instance variables.
  _key = None
  _values = None

  # TODO: Make _ versions of all methods, and make non-_ versions
  # simple aliases. That way the _ version is still accessible even if
  # the non-_ version has been obscured by a property.

  # TODO: Distinguish between purposes: to call FromPb() or setvalue() etc.
  @datastore_rpc._positional(1)
  def __init__(self, key=None, **kwds):
    cls = self.__class__
    if cls is not Model and cls._properties is None:
      FixUpProperties(cls)
    self._key = key
    self._values = {}
    for name, value in kwds.iteritems():
      prop = getattr(cls, name)
      assert isinstance(prop, Property)
      prop.SetValue(self, value)

  def __repr__(self):
    args = []
    if self._key is not None:
      args.append('key=%r' % self._key)
    for name_value in sorted(self._values.iteritems()):
      args.append('%s=%r' % name_value)
    s = '%s(%s)' % (self.__class__.__name__, ', '.join(args))
    return s

  # TODO: Make kind a property also?
  @classmethod
  def getkind(cls):
    return cls.__name__

  def _getkey(self):
    return self._key

  def _setkey(self, key):
    if key is not None:
      assert isinstance(key, Key)
      if self.__class__ is not Model:
        assert list(key.pairs())[-1][0] == self.getkind()
    self._key = key

  def _delkey(self):
    self._key = None

  key = property(_getkey, _setkey, _delkey)

  def __hash__(self):
    raise TypeError('Model is not immutable')

  def __eq__(self, other):
    if other.__class__ is not self.__class__:
      return NotImplemented
    # It's okay to use private names -- we're the same class
    if self._key != other._key:
      # TODO: If one key is None and the other is an explicit
      # incomplete key of the simplest form, this should be OK.
      return False
    # Ignore differences in values that are None.
    self_values = [(name, value)
                   for name, value in self._values.iteritems()
                   if value is not None]
    other_values = [(name, value)
                    for name, value in other._values.iteritems()
                    if value is not None]
    return self_values == other_values

  def __ne__(self, other):
    eq = self.__eq__(other)
    if eq is  NotImplemented:
      return NotImplemented
    return not eq

  def ToPb(self):
    pb = entity_pb.EntityProto()

    # TODO: Move the key stuff into ModelAdapter.entity_to_pb()?
    key = self._key
    if key is None:
      ref = _ReferenceFromPairs([(self.getkind(), None)], pb.mutable_key())
      ref.set_app(_DefaultAppId())
    else:
      ref = key._Key__reference  # Don't copy
      pb.mutable_key().CopyFrom(ref)
    group = pb.mutable_entity_group()
    elem = ref.path().element(0)
    if elem.id() or elem.name():
      group.add_element().CopyFrom(elem)

    if self._properties:
      # TODO: Sort by property declaration order
      for name, prop in sorted(self._properties.iteritems()):
        prop.Serialize(self, pb)

    return pb

  # TODO: Make this a class method?
  def FromPb(self, pb):
    assert not self._key
    assert not self._values
    assert isinstance(pb, entity_pb.EntityProto)

    # TODO: Move the key stuff into ModelAdapter.pb_to_entity()?
    if pb.has_key():
      self._key = Key(reference=pb.key())

    indexed_properties = pb.property_list()
    unindexed_properties = pb.raw_property_list()
    for plist in [indexed_properties, unindexed_properties]:
      for p in plist:
        prop = self.GetPropertyFor(p, plist is indexed_properties)
        prop.Deserialize(self, p)

  def GetPropertyFor(self, p, indexed=True, depth=0):
    db_name = p.name()
    parts = db_name.split('.')
    assert len(parts) > depth, (p.name(), parts, depth)
    next = parts[depth]
    prop = None
    if self._properties:
      prop = self._properties.get(next)
    if prop is None:
      prop = self.FakeProperty(p, next, indexed)
    return prop

  def FakeProperty(self, p, next, indexed=True):
    cls = self.__class__
    if self._properties is cls._properties:
      self._properties = dict(cls._properties or ())

    if p.name() != next and not p.name().endswith('.' + next):
      prop = StructuredProperty(Model, next)
      pid = str(id(prop))
      assert pid not in self._values
      self._values[pid] = Model()
    else:
      prop = GenericProperty(next,
                             repeated=p.multiple(),
                             indexed=indexed)

    prop.FixUp(str(id(prop)))  # Use a unique string as Python name.

{'fraction_non_alphanumeric': 0.10458193114159979, 'fraction_numerical': 0.0008628872206402624, 'mean_word_length': 2.8301387970918706, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4172031', 'n_tokens_mistral': 3659, 'n_tokens_neox': 3590, 'n_words': 1122}