<commit_before>"""Pydisque makes Disque easy to access in python."""

import redis
from redis.exceptions import ConnectionError
from functools import wraps
from itertools import izip_longest
import logging

logger = logging.getLogger(__name__)


class Job(object):

    """Represents a Disque Job."""

    def __init__(self, id, queue_name, payload):
        """Initialize a job."""
        self.id = id
        self.queue_name = queue_name
        self.payload = payload

    def __repr__(self):
        """Make a Job easy to read."""
        return '<Job id:%s queue_name:%s>' % (self.id, self.queue_name)


class Node(object):

    """Represents a Disque Node via host and port."""

    def __init__(self, node_id, host, port, connection):
        """
        Initialize a the Disque Node.

        :param node_id:
        :param host:
        :param port:
        :param connection: redis.Redis connection
        :returns:
        """
        self.node_id = node_id
        self.host = host
        self.port = port
        self.connection = connection

    def __repr__(self):
        """Make Node easy to read."""
        return '<Node %s:%s>' % (self.host, self.port)


class retry(object):

    """retry utility object."""

    def __init__(self, retry_count=2):
        """Initialize retry utility object."""
        self.retry_count = retry_count

    def __call__(self, fn):
        """Function wrapper."""
        @wraps(fn)
        def wrapped_f(*args, **kwargs):
            c = 0
            while c <= self.retry_count:
                try:
                    return fn(*args, **kwargs)
                except:
                    logging.critical("retrying because of this exception - %s",
                                     c)
                    logging.exception("exception to retry ")
                    if c == self.retry_count:
                        raise
                c += 1

        return wrapped_f


class Client(object):

    """
    Client is the Disque Client.

    You can pass in a list of nodes, it will try to connect to
    first if it can't then it will try to connect to second and
    so forth.

    :Example:

    >>> client = Client(['localhost:7711', 'localhost:7712'])
    >>> client.connect()

    """

    def __init__(self, nodes=None):
        """Initalize a client to the specified nodes."""
        if nodes is None:
            nodes = ['localhost:7711']

        self.nodes = {}
        for n in nodes:
            self.nodes[n] = None

        self.connected_node = None

    def connect(self):
        """
        Connect to one of the Disque nodes.

        You can get current connection with connected_node property

        :returns: nothing
        """
        self.connected_node = None
        for i, node in self.nodes.items():
            host, port = i.split(':')
            port = int(port)
            redis_client = redis.Redis(host, port)
            try:
                ret = redis_client.execute_command('HELLO')
                format_version, node_id = ret[0], ret[1]
                others = ret[2:]
                self.nodes[i] = Node(node_id, host, port, redis_client)
                self.connected_node = self.nodes[i]
            except redis.exceptions.ConnectionError:
                pass
        if not self.connected_node:
            raise Exception('couldnt connect to any nodes')
        logger.info("connected to node %s" % self.connected_node)

    def get_connection(self):
        """
        Return current connected_nodes connection.

        :rtype: redis.Redis
        """
        return self.connected_node.connection

    @retry()
    def execute_command(self, *args, **kwargs):
        """Execute a command on the connected server."""
        try:
            return self.get_connection().execute_command(*args, **kwargs)
        except ConnectionError as e:
            logger.warn('trying to reconnect')
            self.connect()
            logger.warn('connected')
            raise

    def _grouper(self, iterable, n, fillvalue=None):
        """Collect data into fixed-length chunks or blocks."""
        args = [iter(iterable)] * n
        return izip_longest(fillvalue=fillvalue, *args)

    def info(self):<commit_msg>Fix izip_longest rename in python3 (fixes #24)<commit_after>"""Pydisque makes Disque easy to access in python."""

import redis
from redis.exceptions import ConnectionError
from functools import wraps
try:
  # Python 3
  from itertools import zip_longest
except ImportError:
  from itertools import izip_longest as zip_longest
import logging

logger = logging.getLogger(__name__)


class Job(object):

    """Represents a Disque Job."""

    def __init__(self, id, queue_name, payload):
        """Initialize a job."""
        self.id = id
        self.queue_name = queue_name
        self.payload = payload

    def __repr__(self):
        """Make a Job easy to read."""
        return '<Job id:%s queue_name:%s>' % (self.id, self.queue_name)


class Node(object):

    """Represents a Disque Node via host and port."""

    def __init__(self, node_id, host, port, connection):
        """
        Initialize a the Disque Node.

        :param node_id:
        :param host:
        :param port:
        :param connection: redis.Redis connection
        :returns:
        """
        self.node_id = node_id
        self.host = host
        self.port = port
        self.connection = connection

    def __repr__(self):
        """Make Node easy to read."""
        return '<Node %s:%s>' % (self.host, self.port)


class retry(object):

    """retry utility object."""

    def __init__(self, retry_count=2):
        """Initialize retry utility object."""
        self.retry_count = retry_count

    def __call__(self, fn):
        """Function wrapper."""
        @wraps(fn)
        def wrapped_f(*args, **kwargs):
            c = 0
            while c <= self.retry_count:
                try:
                    return fn(*args, **kwargs)
                except:
                    logging.critical("retrying because of this exception - %s",
                                     c)
                    logging.exception("exception to retry ")
                    if c == self.retry_count:
                        raise
                c += 1

        return wrapped_f


class Client(object):

    """
    Client is the Disque Client.

    You can pass in a list of nodes, it will try to connect to
    first if it can't then it will try to connect to second and
    so forth.

    :Example:

    >>> client = Client(['localhost:7711', 'localhost:7712'])
    >>> client.connect()

    """

    def __init__(self, nodes=None):
        """Initalize a client to the specified nodes."""
        if nodes is None:
            nodes = ['localhost:7711']

        self.nodes = {}
        for n in nodes:
            self.nodes[n] = None

        self.connected_node = None

    def connect(self):
        """
        Connect to one of the Disque nodes.

        You can get current connection with connected_node property

        :returns: nothing
        """
        self.connected_node = None
        for i, node in self.nodes.items():
            host, port = i.split(':')
            port = int(port)
            redis_client = redis.Redis(host, port)
            try:
                ret = redis_client.execute_command('HELLO')
                format_version, node_id = ret[0], ret[1]
                others = ret[2:]
                self.nodes[i] = Node(node_id, host, port, redis_client)
                self.connected_node = self.nodes[i]
            except redis.exceptions.ConnectionError:
                pass
        if not self.connected_node:
            raise Exception('couldnt connect to any nodes')
        logger.info("connected to node %s" % self.connected_node)

    def get_connection(self):
        """
        Return current connected_nodes connection.

        :rtype: redis.Redis
        """
        return self.connected_node.connection

    @retry()
    def execute_command(self, *args, **kwargs):
        """Execute a command on the connected server."""
        try:
            return self.get_connection().execute_command(*args, **kwargs)
        except ConnectionError as e:
            logger.warn('trying to reconnect')
            self.connect()
            logger.warn('connected')
            raise

    def _grouper(self, iterable, n, fillvalue=None):
        """Collect data into fixed-length chunks or blocks."""
        args = [iter(iterable)] * n
        return zip_longest(fillvalue=fillvalue, *args)

    def info(self):
{'fraction_non_alphanumeric': 0.10515296033500059, 'fraction_numerical': 0.004652785855530999, 'mean_word_length': 1.899831365935919, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3782319', 'n_tokens_mistral': 2409, 'n_tokens_neox': 2309, 'n_words': 720}