<commit_before>#!/usr/bin/env python
"""
base collection classes for building collections of model objects

http://xkcd.com/353/

<NAME>
2013
http://joshashby.com
<EMAIL>
"""
import models.redis.baseRedisModel as bum
import config.config as c


class baseRedisCollection(object):
    """
    Attempts to provide a collection for `redisObjects` providing
    a way to customize the creation of each object, and a way to sort
    the objects by various fields. Inheriting classes should only need
    to override `preInitAppend` and `postInitAppend` currently.

    For an example see `bucketsPail` in `models.bucket.bucketModel`
    """
    def __init__(self, pattern, redis=c.database.redisBucketServer):
        """
        Assumes the Redis objects are stored in a pattern of `what:id:parts``
        Where `what` is like a class, where all keys in `what` are of the same
        object type in the system.

        :param pattern: The pattern which to find all the keys in this collection
            For example, `bucket:*:value` is a general pattern to find all the ids
            in the category of `bucket`

        :param redis: The Redis instance which this collection should use
        """
        self._collection = []
        self.redis = redis
        self.pattern = pattern
        key = pattern.split(":")[0] + ":"
        pail = list(set([ key+item.split(":")[1] for item in self.redis.keys(self.pattern) ]))

        for drop in pail:
            drip = bum.redisObject(drop)
            drip = self.preInitAppend(drip)
            self._collection.append(drip)
            self.postInitAppend()

    def preInitAppend(self, drip):
        """
        Pre append hook for adding a redisObject to the internal
        _collection list. Inheriting classes should override this if
        any modification needs to be made on `drip`

        :param drip: a `redisObject` instance
        :type drip: redisObject

        :return: `drip` instance modified or unmodified
        :rtype: redisObject
        """
        return drip

    def postInitAppend(self):
        """
        Post append hook that runs after each `redisObject` is inserted into
        self._collection

        Note: Accepts nothing and returns nothing.
        """
        pass

    def sortBy(self, by, desc=False):
        """
        Sorts the collection by the field specified in `by`

        :param by: The name of the field by which the collection should be
            sorted by
        :type by: Str

        :param desc: If false then the collection is sorted then revered.
        :type desc: Boolean

        :return: The collection after sorting
        :rtype: List
        """
        self._collection.sort(key=lambda x: x[by])
        if not desc:
            self._collection.reverse()
        return self._collection

    def __iter__(self):
        """
        Emulates an iterator for use in `for` loops and such
        """
        for drip in self._collection:
            yield drip

<commit_msg>Update redis collections to use a list of keys in that collection if available<commit_after>#!/usr/bin/env python
"""
base collection classes for building collections of model objects

http://xkcd.com/353/

<NAME>
2013
http://joshashby.com
<EMAIL>
"""
import models.redis.baseRedisModel as bum
import config.config as c


class baseRedisCollection(object):
    """
    Attempts to provide a collection for `redisObjects` providing
    a way to customize the creation of each object, and a way to sort
    the objects by various fields. Inheriting classes should only need
    to override `preInitAppend` and `postInitAppend` currently.

    For an example see `bucketsPail` in `models.bucket.bucketModel`
    """
    def __init__(self, pattern, redis=c.database.redisBucketServer):
        """
        Assumes the Redis objects are stored in a pattern of `what:id:parts``
        Where `what` is like a class, where all keys in `what` are of the same
        object type in the system.

        It also assumes that this collection is stored in a redis list under
        the `what` name, so for buckets, the list key would be `bucket`

        :param pattern: The pattern which to find all the keys in this collection
            For example, `bucket:*:value` is a general pattern to find all the ids
            in the category of `bucket`

        :param redis: The Redis instance which this collection should use
        """
        self._collection = []
        self.redis = redis
        self.pattern = pattern
        key = pattern.split(":")[0]
        pail = bum.redisList(key)
        if not pail:
            self.pail = bum.redisList(key)
            key += ":"
            pail = list(set([ key+item.split(":")[1] for item in self.redis.keys(self.pattern) ]))

        for drop in pail:
            if type(pail) == bum.redisList:
                drip = bum.redisObject(drop)
            else:
                drip = bum.redisObject(drop)
                self.pail.append(drop)

            drip = self.preInitAppend(drip)
            self._collection.append(drip)
            self.postInitAppend()

    def preInitAppend(self, drip):
        """
        Pre append hook for adding a redisObject to the internal
        _collection list. Inheriting classes should override this if
        any modification needs to be made on `drip`

        :param drip: a `redisObject` instance
        :type drip: redisObject

        :return: `drip` instance modified or unmodified
        :rtype: redisObject
        """
        return drip

    def postInitAppend(self):
        """
        Post append hook that runs after each `redisObject` is inserted into
        self._collection

        Note: Accepts nothing and returns nothing.
        """
        pass

    def sortBy(self, by, desc=False):
        """
        Sorts the collection by the field specified in `by`

        :param by: The name of the field by which the collection should be
            sorted by
        :type by: Str

        :param desc: If false then the collection is sorted then revered.
        :type desc: Boolean

        :return: The collection after sorting
        :rtype: List
        """
        self._collection.sort(key=lambda x: x[by])
        if not desc:
            self._collection.reverse()
        return self._collection

    def addObject(self, key):
        self.pail.append("%s:%s"%(self.patter.split(":")[0], key))
        pass

    def delObject(self, key):
        self.pail.remove("%s:%s"%(self.pattern.split(":")[0], key))
        pass

    def update(self, key):
        key = self.pattern.split(":")[0] + ":"
        pail = [ key+item.split(":")[1] for item in self.redis.keys(self.pattern) ]
        setPail = set(pail)

        setSelfPail = set(self.pail)

        difference = setSelfPail.difference(setPail)

        for drop in difference:
            self.pail.append(drop)

    def __iter__(self):
        """
        Emulates an iterator for use in `for` loops and such
        """
        for drip in self._collection:
            yield drip


{'fraction_non_alphanumeric': 0.08782350445481545, 'fraction_numerical': 0.003111299674727761, 'mean_word_length': 2.2145454545454544, 'pattern_counts': {'":': 11, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '969618', 'n_tokens_mistral': 2001, 'n_tokens_neox': 1932, 'n_words': 721}