<commit_before>import logging
import random
import traceback

import eventlet
import msgpack
import redis
import routes

from turnstile import limits
from turnstile import utils


LOG = logging.getLogger('turnstile')


class TurnstileRedis(redis.StrictRedis):
    def safe_update(self, key, klass, update, *args):
        """
        Safely creates or updates an object in the database.

        :param key: The key the object should be looked up under.
        :param klass: The Python class corresponding to the object.
        :param update: A callable taking a single argument--the object
                       being updated.  The return value of this
                       callable will become the return value of the
                       safe_update() function.  Note that the callable
                       may be called multiple times for a single
                       update.

        If the object does not currently exist in the database, its
        constructor will be called with the database as the first
        parameter, followed by the remaining positional arguments.  If the
        object does currently exist in the database, its hydrate() class
        method will be called, again with the database as the first
        parameter, followed by the remaining positional arguments,
        followed by a dictionary.

        Once the object is built, the update function is called; its
        return value will be saved and the object will be serialized back
        into the database (the object must have a dehydrate() instance
        method taking no parameters and returning a dictionary).  If the
        object has an 'expire' attribute, the key will be set to expire at
        the time given by 'expire'.

        Note that if the key is updated before processing is complete,
        the object will be reloaded and the update function called
        again.  This ensures that the update is atomic, but means that
        the update function must have no side effects other than
        changes to the object it is passed.
        """

        # Do this in a transaction
        with self.pipeline() as pipe:
            while True:
                try:
                    # Watch for changes to the key
                    pipe.watch(key)

                    # Look up or create the object
                    raw = pipe.get(key)
                    if raw is None:
                        obj = klass(self, *args)
                    else:
                        obj = klass.hydrate(self, msgpack.loads(raw), *args)

                    # Start the transaction...
                    pipe.multi()

                    # Call the processor...
                    result = update(obj)

                    # Save the object back to the database
                    pipe.set(key, msgpack.dumps(obj.dehydrate()))

                    # Set its expiration time
                    try:
                        pipe.expireat(key, obj.expire)
                    except AttributeError:
                        # No expiration time available
                        pass

                    # Execute the transaction
                    pipe.execute()
                except redis.WatchError:
                    # Try again...
                    continue
                else:
                    # We're all done!
                    break

        return result


def initialize(middleware, config):
    """
    Initialize a connection to the Redis database.
    """

    # Extract relevant connection information from the configuration
    kwargs = {}
    for cfg_var, type_ in [('host', str), ('port', int), ('db', int),
                           ('password', str), ('socket_timeout', int),
                           ('unix_socket_path', str)]:
        if cfg_var in config:
            kwargs[cfg_var] = type_(config[cfg_var])

    # Make sure we have at a minimum the hostname
    if 'host' not in kwargs and 'unix_socket_path' not in kwargs:
        raise redis.ConnectionError("No host specified for redis database")

    # Look up the connection pool configuration
    cpool_class = None
    cpool = {}
    for key, value in config:<commit_msg>Walk the configuration items, not the keys<commit_after>import logging
import random
import traceback

import eventlet
import msgpack
import redis
import routes

from turnstile import limits
from turnstile import utils


LOG = logging.getLogger('turnstile')


class TurnstileRedis(redis.StrictRedis):
    def safe_update(self, key, klass, update, *args):
        """
        Safely creates or updates an object in the database.

        :param key: The key the object should be looked up under.
        :param klass: The Python class corresponding to the object.
        :param update: A callable taking a single argument--the object
                       being updated.  The return value of this
                       callable will become the return value of the
                       safe_update() function.  Note that the callable
                       may be called multiple times for a single
                       update.

        If the object does not currently exist in the database, its
        constructor will be called with the database as the first
        parameter, followed by the remaining positional arguments.  If the
        object does currently exist in the database, its hydrate() class
        method will be called, again with the database as the first
        parameter, followed by the remaining positional arguments,
        followed by a dictionary.

        Once the object is built, the update function is called; its
        return value will be saved and the object will be serialized back
        into the database (the object must have a dehydrate() instance
        method taking no parameters and returning a dictionary).  If the
        object has an 'expire' attribute, the key will be set to expire at
        the time given by 'expire'.

        Note that if the key is updated before processing is complete,
        the object will be reloaded and the update function called
        again.  This ensures that the update is atomic, but means that
        the update function must have no side effects other than
        changes to the object it is passed.
        """

        # Do this in a transaction
        with self.pipeline() as pipe:
            while True:
                try:
                    # Watch for changes to the key
                    pipe.watch(key)

                    # Look up or create the object
                    raw = pipe.get(key)
                    if raw is None:
                        obj = klass(self, *args)
                    else:
                        obj = klass.hydrate(self, msgpack.loads(raw), *args)

                    # Start the transaction...
                    pipe.multi()

                    # Call the processor...
                    result = update(obj)

                    # Save the object back to the database
                    pipe.set(key, msgpack.dumps(obj.dehydrate()))

                    # Set its expiration time
                    try:
                        pipe.expireat(key, obj.expire)
                    except AttributeError:
                        # No expiration time available
                        pass

                    # Execute the transaction
                    pipe.execute()
                except redis.WatchError:
                    # Try again...
                    continue
                else:
                    # We're all done!
                    break

        return result


def initialize(middleware, config):
    """
    Initialize a connection to the Redis database.
    """

    # Extract relevant connection information from the configuration
    kwargs = {}
    for cfg_var, type_ in [('host', str), ('port', int), ('db', int),
                           ('password', str), ('socket_timeout', int),
                           ('unix_socket_path', str)]:
        if cfg_var in config:
            kwargs[cfg_var] = type_(config[cfg_var])

    # Make sure we have at a minimum the hostname
    if 'host' not in kwargs and 'unix_socket_path' not in kwargs:
        raise redis.ConnectionError("No host specified for redis database")

    # Look up the connection pool configuration
    cpool_class = None
    cpool = {}
    for key, value in config.items():
{'fraction_non_alphanumeric': 0.055311572700296734, 'fraction_numerical': 0.0, 'mean_word_length': 1.6265586034912718, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '621933', 'n_tokens_mistral': 1964, 'n_tokens_neox': 1861, 'n_words': 883}