<commit_before>import itertools

from vumi.persist.model import VumiRiakError
from go_api.collections.errors import CollectionUsageError
from go_api.queue import PausingQueueCloseMarker
from twisted.internet.defer import inlineCallbacks, returnValue


@inlineCallbacks
def _get_page_of_keys(
        model_proxy, user_account_key, max_results, cursor,
        field_name='user_account'):
    try:
        contact_keys = yield model_proxy.index_keys_page(
            field_name, user_account_key, max_results=max_results,
            continuation=cursor)
    except VumiRiakError:
        raise CollectionUsageError(
            "Riak error, possible invalid cursor: %r" % (cursor,))

    cursor = contact_keys.continuation
    returnValue((cursor, contact_keys))


@inlineCallbacks
def _fill_queue(q, get_page, get_dict, close_queue=True):
    keys_deferred = get_page(None)

    while True:
        cursor, keys = yield keys_deferred
        if cursor is not None:
            # Get the next page of keys while we fetch the objects
            keys_deferred = get_page(cursor)

        for key in keys:
            obj = yield get_dict(key)
            yield q.put(obj)

        if cursor is None:
            break

    if close_queue:
        q.put(PausingQueueCloseMarker())


@inlineCallbacks
def _get_smart_page_of_keys(
        model_proxy, max_results, cursor, query):
    contact_keys = yield model_proxy.real_search(query)
    contact_keys.sort()
    cursor = None if cursor == '' else cursor
    page = list(itertools.islice(
        itertools.dropwhile(lambda key: key <= cursor, contact_keys),
        max_results))
    if len(page) == max_results:
        new_cursor = page[-1]
    else:
        new_cursor = None
    returnValue((new_cursor, page))<commit_msg>Changed pagination to only get necessary keys when fetching contacts for a smart group.<commit_after>import itertools

from vumi.persist.model import VumiRiakError
from go_api.collections.errors import CollectionUsageError
from go_api.queue import PausingQueueCloseMarker
from twisted.internet.defer import inlineCallbacks, returnValue


@inlineCallbacks
def _get_page_of_keys(
        model_proxy, user_account_key, max_results, cursor,
        field_name='user_account'):
    try:
        contact_keys = yield model_proxy.index_keys_page(
            field_name, user_account_key, max_results=max_results,
            continuation=cursor)
    except VumiRiakError:
        raise CollectionUsageError(
            "Riak error, possible invalid cursor: %r" % (cursor,))

    cursor = contact_keys.continuation
    returnValue((cursor, contact_keys))


@inlineCallbacks
def _fill_queue(q, get_page, get_dict, close_queue=True):
    keys_deferred = get_page(None)

    while True:
        cursor, keys = yield keys_deferred
        if cursor is not None:
            # Get the next page of keys while we fetch the objects
            keys_deferred = get_page(cursor)

        for key in keys:
            obj = yield get_dict(key)
            yield q.put(obj)

        if cursor is None:
            break

    if close_queue:
        q.put(PausingQueueCloseMarker())


@inlineCallbacks
def _get_smart_page_of_keys(
        model_proxy, max_results, cursor, query):
    contact_keys = yield model_proxy.real_search(
        query, rows=max_results, start=cursor)
    if len(contact_keys) == 0:
        new_cursor = None
    else:
        new_cursor = cursor + len(contact_keys)
    returnValue((new_cursor, contact_keys))
{'fraction_non_alphanumeric': 0.07124463519313305, 'fraction_numerical': 0.0005722460658082976, 'mean_word_length': 3.0651162790697675, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '217450', 'n_tokens_mistral': 1039, 'n_tokens_neox': 1021, 'n_words': 269}