<commit_before># -*- coding: utf-8 -*-

"""
Query classes for Tokyo Tyrant API implementation.
"""

import copy
import warnings

from protocol import TyrantProtocol
import utils


CACHE_CHUNK_SIZE = 1000


class Query(object):
    """
    A lazy abstraction for queries via Tyrant protocol.

    You will normally instantiate Query this way::

        >>> from pyrant import Tyrant
        >>> t = Tyrant(host='localhost', port=1983)
        >>> query = t.query

    .. note:: the results are cached in two ways. First, the complete list of
        relevant keys is fetched and stored in the query object. Second, the
        corresponding data is fetched in large chunks depending on what slices
        or indices you request. Sometimes the chunks are not large enough and
        we hit the database too many times. To minimize the overhead you may
        want to increase the chunk size. You can use
        :meth:`~pyrant.query.Query.set_chunk_size` for that purpose.

    """

    def __init__(self, proto, db_type, literal=False, conditions=None,
                 columns=None, ms_type=None, ms_conditions=None):
        if conditions:
            assert isinstance(conditions, list) and \
                   all(isinstance(c, Condition) for c in conditions), \
                   'Expected a list of Condition instances, got %s' % conditions
        self.literal = literal
        self._conditions = conditions or []
        self._ordering = Ordering()
        self._proto = proto
        self._db_type = db_type
        self._columns = columns
        self._ms_type = ms_type
        self._ms_conditions = ms_conditions

        # cache
        self._cache = ResultCache(self)

    #
    # PYTHON MAGIC METHODS
    #

    def __and__(self, other):
        return self.intersect(other)

    def __contains__(self, key):
        keys = self._cache.get_keys(self._do_search)
        return key in keys

    def __getitem__(self, k):
        # Retrieve an item or slice from the set of results.

        # XXX do we really need to cache the data? What if there are a couple
        #     millions items and the user wants to just iterate over them in
        #     order to calculate an aggregate? I think cache should be either
        #     kept small or turned off by default or completely removed.
        #     The user can always easily cache the data explicitly by keeping
        #     the results in a variable (i.e. "records = query[:]").

        if isinstance(k, slice):
            return self._get_slice(k)
        elif isinstance(k, (int, long)):
            return self._get_item(k)
        else:
            raise TypeError("Query indices must be integers")

    def _get_slice(self, s):
        # Check slice integrity    XXX check if this is still resonable
        for x in s.start, s.stop:
            if x is not None and x < 0:
                raise ValueError('Negative indexing is not supported')
        if s.start and s.start == s.stop:
            raise ValueError('Zero-length slices are not supported')

        # retrieve and cache keys
        self._cache.get_keys(self._do_search)

        items = self._cache.get_items(s.start or 0, s.stop)
        return list(items)

    def _get_item(self, index):
        if index < 0:
            raise ValueError('Negative indexing is not supported')

        # retrieve and cache keys
        self._cache.get_keys(self._do_search)

        item = self._cache.get_item(index)
        if item is None:
            raise IndexError
        return item

    def __len__(self):
        return len(self[:])

    def __or__(self, other):
        return self.union(other)

    def __repr__(self):
        # Do the query using getitem
        return str(self[:])

    def __sub__(self, other):
        return self.minus(other)

    #
    # PRIVATE METHODS
    #

    def _add_to_metasearch(self, other, operator):
        """
        Returns a new Query object resulting from mapping `self` with ``other``
        by applying the given ``operator`` which is one of the operators defined
        in Tokyo Tyrant protocol: `TyrantProtocol.TDBMSUNION`,
        `TyrantProtocol.TDBMSISECT` or `TyrantProtocol.TDBMSDIFF`.
        """
        query = self._clone()
        assert isinstance(other, Query), "This function needs other Query object type"
        assert query._ms_type in (None, operator), "You can not mix union with intersect or minus"
        if query._ms_conditions is None:
            query._ms_conditions = []
        other = other._clone()
        query._ms_conditions.append(other._conditions)
        query._ms_type = operator
        return query

    def _clone(self):
        defaults = {
            'literal': self.literal,
            'conditions': [c._clone() for c in self._conditions],
            'ms_type': self._ms_type,
        }

        if self._ms_conditions:
            defaults.update(
                ms_conditions = [[query._clone() for query in conds]
                                    for conds in self._ms_conditions],
            )

        if self._columns:
            defaults.update(columns=self._columns[:])

        return Query(self._proto, self._db_type, **defaults)

    def _do_search(self, conditions=None, limit=None, offset=None,
                   out=False, count=False, hint=False, columns=None):
        """
        Returns keys of items that correspond to the Query instance.
        """
        defaults = {
            'out': out,
            'count': count,
            'hint': hint,
            'conditions': conditions or [c.prepare() for c in self._conditions],
            'limit': limit,
            'offset': offset,
        }
        if columns:
            defaults.update(columns=columns[:])
        if self._ordering:
            defaults.update(
                order_column = self._ordering.name,
                order_type   = self._ordering.type,
            )
        if self._ms_conditions:
            # update search conditions with metaseach conditions
            defaults.update(
                ms_type = self._ms_type,
                ms_conditions = [
                    [condition.prepare() for condition in metasearch_conditions]
                    for metasearch_condictions in self._ms_conditions
                ]
            )

        return self._proto.search(**defaults)

    def _filter(self, negate, args, kwargs):
        query = self._clone()

        # Iterate arguments. Should be instances of Q
        for cond in args:
            assert isinstance(cond, Condition), "Arguments must be instances of Q"
            c = cond._clone()
            c.negate = c.negate ^ negate
            query._conditions.append(c)

        # Generate Condition objects with arguments as needed
        for name, expr in kwargs.iteritems():
            c = Condition(name, expr)
            c.negate = negate
            query._conditions.append(c)

        return query

    def _to_python(self, elem):
        return utils.to_python(elem, self._db_type)
<commit_msg>BUGFIX: misprint (catched by nosetests).<commit_after># -*- coding: utf-8 -*-

"""
Query classes for Tokyo Tyrant API implementation.
"""

import copy
import warnings

from protocol import TyrantProtocol
import utils


CACHE_CHUNK_SIZE = 1000


class Query(object):
    """
    A lazy abstraction for queries via Tyrant protocol.

    You will normally instantiate Query this way::

        >>> from pyrant import Tyrant
        >>> t = Tyrant(host='localhost', port=1983)
        >>> query = t.query

    .. note:: the results are cached in two ways. First, the complete list of
        relevant keys is fetched and stored in the query object. Second, the
        corresponding data is fetched in large chunks depending on what slices
        or indices you request. Sometimes the chunks are not large enough and
        we hit the database too many times. To minimize the overhead you may
        want to increase the chunk size. You can use
        :meth:`~pyrant.query.Query.set_chunk_size` for that purpose.

    """

    def __init__(self, proto, db_type, literal=False, conditions=None,
                 columns=None, ms_type=None, ms_conditions=None):
        if conditions:
            assert isinstance(conditions, list) and \
                   all(isinstance(c, Condition) for c in conditions), \
                   'Expected a list of Condition instances, got %s' % conditions
        self.literal = literal
        self._conditions = conditions or []
        self._ordering = Ordering()
        self._proto = proto
        self._db_type = db_type
        self._columns = columns
        self._ms_type = ms_type
        self._ms_conditions = ms_conditions

        # cache
        self._cache = ResultCache(self)

    #
    # PYTHON MAGIC METHODS
    #

    def __and__(self, other):
        return self.intersect(other)

    def __contains__(self, key):
        keys = self._cache.get_keys(self._do_search)
        return key in keys

    def __getitem__(self, k):
        # Retrieve an item or slice from the set of results.

        # XXX do we really need to cache the data? What if there are a couple
        #     millions items and the user wants to just iterate over them in
        #     order to calculate an aggregate? I think cache should be either
        #     kept small or turned off by default or completely removed.
        #     The user can always easily cache the data explicitly by keeping
        #     the results in a variable (i.e. "records = query[:]").

        if isinstance(k, slice):
            return self._get_slice(k)
        elif isinstance(k, (int, long)):
            return self._get_item(k)
        else:
            raise TypeError("Query indices must be integers")

    def _get_slice(self, s):
        # Check slice integrity    XXX check if this is still resonable
        for x in s.start, s.stop:
            if x is not None and x < 0:
                raise ValueError('Negative indexing is not supported')
        if s.start and s.start == s.stop:
            raise ValueError('Zero-length slices are not supported')

        # retrieve and cache keys
        self._cache.get_keys(self._do_search)

        items = self._cache.get_items(s.start or 0, s.stop)
        return list(items)

    def _get_item(self, index):
        if index < 0:
            raise ValueError('Negative indexing is not supported')

        # retrieve and cache keys
        self._cache.get_keys(self._do_search)

        item = self._cache.get_item(index)
        if item is None:
            raise IndexError
        return item

    def __len__(self):
        return len(self[:])

    def __or__(self, other):
        return self.union(other)

    def __repr__(self):
        # Do the query using getitem
        return str(self[:])

    def __sub__(self, other):
        return self.minus(other)

    #
    # PRIVATE METHODS
    #

    def _add_to_metasearch(self, other, operator):
        """
        Returns a new Query object resulting from mapping `self` with ``other``
        by applying the given ``operator`` which is one of the operators defined
        in Tokyo Tyrant protocol: `TyrantProtocol.TDBMSUNION`,
        `TyrantProtocol.TDBMSISECT` or `TyrantProtocol.TDBMSDIFF`.
        """
        query = self._clone()
        assert isinstance(other, Query), "This function needs other Query object type"
        assert query._ms_type in (None, operator), "You can not mix union with intersect or minus"
        if query._ms_conditions is None:
            query._ms_conditions = []
        other = other._clone()
        query._ms_conditions.append(other._conditions)
        query._ms_type = operator
        return query

    def _clone(self):
        defaults = {
            'literal': self.literal,
            'conditions': [c._clone() for c in self._conditions],
            'ms_type': self._ms_type,
        }

        if self._ms_conditions:
            defaults.update(
                ms_conditions = [[query._clone() for query in conds]
                                    for conds in self._ms_conditions],
            )

        if self._columns:
            defaults.update(columns=self._columns[:])

        return Query(self._proto, self._db_type, **defaults)

    def _do_search(self, conditions=None, limit=None, offset=None,
                   out=False, count=False, hint=False, columns=None):
        """
        Returns keys of items that correspond to the Query instance.
        """
        defaults = {
            'out': out,
            'count': count,
            'hint': hint,
            'conditions': conditions or [c.prepare() for c in self._conditions],
            'limit': limit,
            'offset': offset,
        }
        if columns:
            defaults.update(columns=columns[:])
        if self._ordering:
            defaults.update(
                order_column = self._ordering.name,
                order_type   = self._ordering.type,
            )
        if self._ms_conditions:
            # update search conditions with metaseach conditions
            defaults.update(
                ms_type = self._ms_type,
                ms_conditions = [
                    [condition.prepare() for condition in metasearch_conditions]
                    for metasearch_conditions in self._ms_conditions
                ]
            )

        return self._proto.search(**defaults)

    def _filter(self, negate, args, kwargs):
        query = self._clone()

        # Iterate arguments. Should be instances of Q
        for cond in args:
            assert isinstance(cond, Condition), "Arguments must be instances of Q"
            c = cond._clone()
            c.negate = c.negate ^ negate
            query._conditions.append(c)

        # Generate Condition objects with arguments as needed
        for name, expr in kwargs.iteritems():
            c = Condition(name, expr)
            c.negate = negate
            query._conditions.append(c)

        return query

    def _to_python(self, elem):
        return utils.to_python(elem, self._db_type)

{'fraction_non_alphanumeric': 0.0823629650667424, 'fraction_numerical': 0.0017040613462084636, 'mean_word_length': 1.9790609137055837, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7610678', 'n_tokens_mistral': 3788, 'n_tokens_neox': 3612, 'n_words': 1264}