<commit_before>import logging
import redis
import urlparse
import dill as pickle
from dill.detect import errors
from will import settings


class StorageMixin(object):
    def bootstrap_storage(self):
        # redis://localhost:6379/7
        # or
        # redis://rediscloud:asdfkjaslkdjflasdf@pub-redis-12345.us-east-1-1.2.ec2.garantiadata.com:12345
        url = urlparse.urlparse(settings.WILL_REDIS_URL)

        if hasattr(url, "path"):
            db = url.path[1:]
        else:
            db = 0

        self.storage = redis.Redis(host=url.hostname, port=url.port, db=db, password=url.password)

    def save(self, key, value):
        if hasattr(self, "bot"):
            storage = self.bot.storage
        elif not hasattr(self, "storage"):
            self.bootstrap_storage()
            storage = self.storage
        print "saving %s...." % key
        print self.storage

        try:
            ret = self.storage.set(key, pickle.dumps(value))
            # self.storage.save()
            # This really shouldn't be needed, but without it, a subsequent load fails.
            _ = self.storage.get(key)
            # self.storage.save()
            print ret
            if _ is not None:
                print "_ is not none"
                print type(_)
                print "pickle.loads(_) %s" % pickle.loads(_)
            return ret
        except:
            import traceback; traceback.print_exc();
            print errors(value)
            print "Unable to save %s" % key
            logging.warn("Unable to save %s" % key)
            print _


    def clear(self, key):
        if not hasattr(self, "storage"):
            self.bootstrap_storage()
        return self.storage.delete(key)

    def clear_all_keys(self):
        if not hasattr(self, "storage"):
            self.bootstrap_storage()
        return self.storage.flushdb()

    def load(self, key, default=None):
        if not hasattr(self, "storage"):
            self.bootstrap_storage()

        try:
            val = self.storage.get(key)<commit_msg>more refactoring..<commit_after>import logging
import redis
import urlparse
import dill as pickle
from dill.detect import errors
from will import settings


class StorageMixin(object):
    def bootstrap_storage(self):
        if hasattr(self, "bot"):
            self.storage = self.bot.storage
        elif not hasattr(self, "storage"):
            # redis://localhost:6379/7
            # or
            # redis://rediscloud:asdfkjaslkdjflasdf@pub-redis-12345.us-east-1-1.2.ec2.garantiadata.com:12345
            url = urlparse.urlparse(settings.WILL_REDIS_URL)

            if hasattr(url, "path"):
                db = url.path[1:]
            else:
                db = 0

            self.storage = redis.Redis(host=url.hostname, port=url.port, db=db, password=<PASSWORD>)

    def save(self, key, value):
        
        if not hasattr(self, "storage"):
            self.bootstrap_storage()

        print "saving %s...." % key

        try:
            ret = self.storage.set(key, pickle.dumps(value))
            # self.storage.save()
            # This really shouldn't be needed, but without it, a subsequent load fails.
            _ = self.storage.get(key)
            # self.storage.save()
            print ret
            if _ is not None:
                print "_ is not none"
                print type(_)
                print "pickle.loads(_) %s" % pickle.loads(_)
            return ret
        except:
            import traceback; traceback.print_exc();
            "print errors(value):"
            print errors(value)
            print "Unable to save %s" % key
            logging.warn("Unable to save %s" % key)
            print _


    def clear(self, key):
        if not hasattr(self, "storage"):
            self.bootstrap_storage()
        return self.storage.delete(key)

    def clear_all_keys(self):
        if not hasattr(self, "storage"):
            self.bootstrap_storage()
        return self.storage.flushdb()

    def load(self, key, default=None):
        if not hasattr(self, "storage"):
            self.bootstrap_storage()

        try:
            val = self.storage.get(key)
{'fraction_non_alphanumeric': 0.10760401721664276, 'fraction_numerical': 0.010043041606886656, 'mean_word_length': 1.8054996646545942, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1036380', 'n_tokens_mistral': 1252, 'n_tokens_neox': 1204, 'n_words': 298}