<commit_before>from .. import mappings
from .filtering import FilterableMetaclass

from future.utils import with_metaclass
from datetime import datetime, timedelta
import time

# The interval to reload volatile properties
volatile_refresh_timeout = timedelta(seconds=15)

class Property:
    def __init__(self, mutable=False, identifier=False, volatile=False, relationship=None, \
            derived_class=None, is_datetime=False, filterable=False):
        self.mutable = mutable
        self.identifier = identifier
        self.volatile = volatile
        self.relationship = relationship
        self.derived_class = derived_class
        self.is_datetime = is_datetime
        self.filterable = filterable

class MappedObject:
    def __init__(self, **vals):
        self._expand_vals(self.__dict__, **vals)

    def _expand_vals(self, target, **vals):
        for v in vals:
            if type(vals[v]) is dict:
                vals[v] = MappedObject(**vals[v])
            elif type(vals[v]) is list:
                # oh mama
                vals[v] = [ MappedObject(**i) if type(i) is dict else i for i in vals[v] ]
        target.update(vals)

    def __repr__(self):
        return "Mapping containing {}".format(vars(self).keys())

class Base(object, with_metaclass(FilterableMetaclass)):
    """
    The Base class knows how to look up api properties of a model, and lazy-load them.
    """
    properties = {}

    def __init__(self, client, id):
        self._set('_populated', False)
        self._set('_last_updated', datetime.min)
        self._set('_client', client)

        for prop in type(self).properties:
            self._set(prop, None)

        self._set('id', id)
        if hasattr(type(self), 'id_attribute'):
            self._set(getattr(type(self), 'id_attribute'), id)

    def __getattribute__(self, name):
        if name in type(self).properties.keys():
            if type(self).properties[name].identifier:
                pass # don't load identifiers from the server, we have those
            elif (object.__getattribute__(self, name) is None and not self._populated \
                    or type(self).properties[name].derived_class) \
                    or (type(self).properties[name].volatile \
                    and object.__getattribute__(self, '_last_updated')
                    + volatile_refresh_timeout < datetime.now()):
                if type(self).properties[name].derived_class:
                    #load derived object(s)
                    self._set(name, type(self).properties[name].derived_class
                            ._api_get_derived(self, getattr(self, '_client')))
                else:
                    self._api_get()

        return object.__getattribute__(self, name)

    def __repr__(self):
        return "{}: {}".format(type(self).__name__, self.id)

    def __setattr__(self, name, value):
        if name in type(self).properties.keys() and not type(self).properties[name].mutable:
            raise AttributeError("'{}' is not a mutable field of '{}'"
                .format(name, type(self).__name__))
        self._set(name, value)

    def save(self):
        resp = self._client.put(type(self).api_endpoint, model=self,
            data=self._serialize())

        if 'error' in resp:
            return False
        return True

    def delete(self):
        resp = self._client.delete(type(self).api_endpoint, model=self)

        if 'error' in resp:
            return False
        self.invalidate()
        return True

    def invalidate(self):
        for key in (k for k in type(self).properties.keys()
            if not type(self).properties[k].identifier):
                self._set(key, None)

        self._populated = False

    def _serialize(self):
        result = { a: getattr(self, a) for a in type(self).properties
            if type(self).properties[a].mutable }

        for k, v in result.items():
            if isinstance(v, Base):
                result[k] = v.id

        return result

    def _api_get(self):
        json = self._client.get(type(self).api_endpoint, model=self)
        self._populate(json)

    def _populate(self, json):
        if not json:
            return

        for key in json:
            if key in (k for k in type(self).properties.keys()
                    if not type(self).properties[k].identifier):
                if type(self).properties[key].relationship  \
                    and not json[key] is None:
                    if isinstance(json[key], list):
                        objs = []
                        for d in json[key]:
                            if not 'id' in d:
                                continue
                            obj = mappings.make(d['id'], getattr(self,'_client'),
                                    cls=type(self).properties[key].relationship)
                            if obj:
                                obj._populate(d)
                            objs.append(obj)
                        self._set(key, objs)
                    else:
                        if isinstance(json[key], dict):
                            related_id = json[key]['id']
                        else:
                            related_id = json[key]
                        obj = mappings.make(related_id, getattr(self,'_client'),
                                cls=type(self).properties[key].relationship)
                        if obj and isinstance(json[key], dict):
                            obj._populate(json[key])
                        self._set(key, obj)
                elif type(json[key]) is dict:
                    self._set(key, MappedObject(**json[key]))
                elif type(json[key]) is list:
                    mapping = MappedObject(_list=json[key])
                    self._set(key, mapping._list)
                elif type(self).properties[key].is_datetime:
                    try:
                        t = time.strptime(json[key], "%Y-%m-%dT%H:%M:%S")
                        self._set(key, datetime.fromtimestamp(time.mktime(t)))
                    except:
                        #TODO - handle this better (or log it?)
                        self._set(key, json[key])
                else:
                    self._set(key, json[key])

        self._set('_populated', True)
        self._set('_last_updated', datetime.now())

    def _set(self, name, value):
        object.__setattr__(self, name, value)

    @classmethod
    def api_list(cls):
        return '/'.join(cls.api_endpoint.split('/')[:-1])
<commit_msg>Adds id_relationships and documents linode.objects.base<commit_after>from .. import mappings
from .filtering import FilterableMetaclass

from future.utils import with_metaclass
from datetime import datetime, timedelta
import time

# The interval to reload volatile properties
volatile_refresh_timeout = timedelta(seconds=15)

class Property:
    def __init__(self, mutable=False, identifier=False, volatile=False, relationship=None, \
            derived_class=None, is_datetime=False, filterable=False, id_relationship=False):
        """
        A Property is an attribute returned from the API, and defines metadata
        about that value.  These are expected to be used as the values of a
        class-level dict named 'properties' in subclasses of Base.

        mutable - This Property should be sent in a call to save()
        identifier - This Property identifies the object in the API
        volatile - Re-query for this Property if the local value is older than the
            volatile refresh timeout
        relationship - The API Object this Property represents
        derived_class - The sub-collection type this Property represents
        is_datetime - True if this Property should be parsed as a datetime.datetime
        filterable - True if the API allows filtering on this property
        id_relationship - This Property should create a relationship with this key as the ID
            (This should be used on fields ending with '_id' only)
        """
        self.mutable = mutable
        self.identifier = identifier
        self.volatile = volatile
        self.relationship = relationship
        self.derived_class = derived_class
        self.is_datetime = is_datetime
        self.filterable = filterable
        self.id_relationship = id_relationship

class MappedObject:
    """
    Converts a dict into values accessible with the dot notation.

    object = {
        "this": "that"
    }

    becomes

    object.this # "that"
    """
    def __init__(self, **vals):
        self._expand_vals(self.__dict__, **vals)

    def _expand_vals(self, target, **vals):
        for v in vals:
            if type(vals[v]) is dict:
                vals[v] = MappedObject(**vals[v])
            elif type(vals[v]) is list:
                # oh mama
                vals[v] = [ MappedObject(**i) if type(i) is dict else i for i in vals[v] ]
        target.update(vals)

    def __repr__(self):
        return "Mapping containing {}".format(vars(self).keys())

class Base(object, with_metaclass(FilterableMetaclass)):
    """
    The Base class knows how to look up api properties of a model, and lazy-load them.
    """
    properties = {}

    def __init__(self, client, id):
        self._set('_populated', False)
        self._set('_last_updated', datetime.min)
        self._set('_client', client)

        for prop in type(self).properties:
            self._set(prop, None)

        self._set('id', id)
        if hasattr(type(self), 'id_attribute'):
            self._set(getattr(type(self), 'id_attribute'), id)

    def __getattribute__(self, name):
        """
        Handles lazy-loading/refreshing an object from the server, and
        getting related objects, as defined in this object's 'properties'
        """
        if name in type(self).properties.keys():
            # We are accessing a Property
            if type(self).properties[name].identifier:
                pass # don't load identifiers from the server, we have those
            elif (object.__getattribute__(self, name) is None and not self._populated \
                    or type(self).properties[name].derived_class) \
                    or (type(self).properties[name].volatile \
                    and object.__getattribute__(self, '_last_updated')
                    + volatile_refresh_timeout < datetime.now()):
                # needs to be loaded from the server
                if type(self).properties[name].derived_class:
                    #load derived object(s)
                    self._set(name, type(self).properties[name].derived_class
                            ._api_get_derived(self, getattr(self, '_client')))
                else:
                    self._api_get()
        elif "{}_id".format(name) in type(self).properties.keys():
            # possible id-based relationship
            related_type = type(self).properties['{}_id'.format(name)].id_relationship
            if related_type:
                # it is a relationship
                relcache_name = '_{}_relcache'.format(name)
                if not hasattr(self, relcache_name):
                    self._set(relcache_name, related_type(self._client, getattr(self, '{}_id'.format(name))))
                return object.__getattribute__(self, relcache_name)

        return object.__getattribute__(self, name)

    def __repr__(self):
        """
        Returns a safe representation of this object without accessing the server
        """
        return "{}: {}".format(type(self).__name__, self.id)

    def __setattr__(self, name, value):
        """
        Enforces allowing editing of only Properties defined as mutable
        """
        if name in type(self).properties.keys() and not type(self).properties[name].mutable:
            raise AttributeError("'{}' is not a mutable field of '{}'"
                .format(name, type(self).__name__))
        self._set(name, value)

    def save(self):
        """
        Send this object's mutable values to the server in a PUT request
        """
        resp = self._client.put(type(self).api_endpoint, model=self,
            data=self._serialize())

        if 'error' in resp:
            return False
        return True

    def delete(self):
        """
        Sends a DELETE request for this object
        """
        resp = self._client.delete(type(self).api_endpoint, model=self)

        if 'error' in resp:
            return False
        self.invalidate()
        return True

    def invalidate(self):
        """
        Invalidates all non-identifier Properties this object has locally,
        causing the next access to re-fetch them from the server
        """
        for key in (k for k in type(self).properties.keys()
            if not type(self).properties[k].identifier):
                self._set(key, None)

        self._populated = False

    def _serialize(self):
        """
        A helper method to build a dict of all mutable Properties of
        this object
        """
        result = { a: getattr(self, a) for a in type(self).properties
            if type(self).properties[a].mutable }

        for k, v in result.items():
            if isinstance(v, Base):
                result[k] = v.id

        return result

    def _api_get(self):
        """
        A helper method to GET this object from the server
        """
        json = self._client.get(type(self).api_endpoint, model=self)
        self._populate(json)

    def _populate(self, json):
        """
        A helper method that, given a JSON object representing this object,
        assigns values based on the properties dict and the attributes of
        its Properties.
        """
        if not json:
            return

        for key in json:
            if key in (k for k in type(self).properties.keys()
                    if not type(self).properties[k].identifier):
                if type(self).properties[key].relationship \
                    and not json[key] is None:
                    if isinstance(json[key], list):
                        objs = []
                        for d in json[key]:
                            if not 'id' in d:
                                continue
                            obj = mappings.make(d['id'], getattr(self,'_client'),
                                    cls=type(self).properties[key].relationship)
                            if obj:
                                obj._populate(d)
                            objs.append(obj)
                        self._set(key, objs)
                    else:
                        if isinstance(json[key], dict):
                            related_id = json[key]['id']
                        else:
                            related_id = json[key]
                        obj = mappings.make(related_id, getattr(self,'_client'),
                                cls=type(self).properties[key].relationship)
                        if obj and isinstance(json[key], dict):
                            obj._populate(json[key])
                        self._set(key, obj)
                elif type(json[key]) is dict:
                    self._set(key, MappedObject(**json[key]))
                elif type(json[key]) is list:
                    mapping = MappedObject(_list=json[key])
                    self._set(key, mapping._list)
                elif type(self).properties[key].is_datetime:
                    try:
                        t = time.strptime(json[key], "%Y-%m-%dT%H:%M:%S")
                        self._set(key, datetime.fromtimestamp(time.mktime(t)))
                    except:
                        #TODO - handle this better (or log it?)
                        self._set(key, json[key])
                else:
                    self._set(key, json[key])

        self._set('_populated', True)
        self._set('_last_updated', datetime.now())

    def _set(self, name, value):
        """
        A helper method to set values of Properties without invoking
        the overloaded __setattr__
        """
        object.__setattr__(self, name, value)

    @classmethod
    def api_list(cls):
        """
        Returns a URL that will produce a list of JSON objects
        of this class' type
        """
        return '/'.join(cls.api_endpoint.split('/')[:-1])

{'fraction_non_alphanumeric': 0.10077709110934345, 'fraction_numerical': 0.00036712965795753535, 'mean_word_length': 1.743662917575961, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2845416', 'n_tokens_mistral': 4355, 'n_tokens_neox': 4141, 'n_words': 1181}