<commit_before>"""Python TAXII 2.1 Client API"""
from __future__ import unicode_literals

import json
import logging
import time

import six
from six.moves.urllib import parse as urlparse

from .. import MEDIA_TYPE_TAXII_V21
from ..common import _filter_kwargs_to_query_params, _TAXIIEndpoint, _to_json
from ..exceptions import AccessError, ValidationError

# Module-level logger
log = logging.getLogger(__name__)


def as_pages(func, per_request=0, *args, **kwargs):
    """Creates a generator for TAXII 2.1 endpoints that support pagination.

    Args:
        func (callable): A v21 function that supports paged requests.
            Currently Get Objects and Get Manifest.
        per_request (int): How many items per request. Default 0.

    Use args or kwargs to pass filter information or other arguments required to make the call.
    """
    envelope = func(limit=per_request, *args, **kwargs)
    yield _to_json(envelope)

    len_objects = len(envelope.get("objects", []))
    if envelope.get("more", False) and len_objects != per_request:
        log.warning("TAXII Server Response with different amount of objects! Setting limit=%s", len_objects)
        per_request = len_objects

    # The while loop will not be executed if the response is received in full.
    while envelope.get("more", False):
        envelope = func(limit=per_request, next=envelope.get("next", ""), *args, **kwargs)
        yield _to_json(envelope)


class Status(_TAXIIEndpoint):
    """TAXII Status Resource.

    This class represents the ``Get Status`` endpoint (section 4.3) and also<commit_msg>minor changes to as_pages()<commit_after>"""Python TAXII 2.1 Client API"""
from __future__ import unicode_literals

import json
import logging
import time

import six
from six.moves.urllib import parse as urlparse

from .. import MEDIA_TYPE_TAXII_V21
from ..common import (
    _filter_kwargs_to_query_params, _grab_total_items_from_resource,
    _TAXIIEndpoint, _to_json
)
from ..exceptions import AccessError, ValidationError

# Module-level logger
log = logging.getLogger(__name__)


def as_pages(func, per_request=0, *args, **kwargs):
    """Creates a generator for TAXII 2.1 endpoints that support pagination.

    Args:
        func (callable): A v21 function that supports paged requests.
            Currently Get Objects and Get Manifest.
        per_request (int): How many items per request. Default 0.

    Use args or kwargs to pass filter information or other arguments required to make the call.
    """
    envelope = func(limit=per_request, *args, **kwargs)
    yield _to_json(envelope)

    total_obtained = _grab_total_items_from_resource(envelope)
    if envelope.get("more", False) and total_obtained != per_request:
        log.warning("TAXII Server Response with different amount of objects! Setting limit=%s", total_obtained)
        per_request = total_obtained

    # The while loop will not be executed if the response is received in full.
    while envelope.get("more", False):
        envelope = func(limit=per_request, next=envelope.get("next", ""), *args, **kwargs)
        yield _to_json(envelope)


class Status(_TAXIIEndpoint):
    """TAXII Status Resource.

    This class represents the ``Get Status`` endpoint (section 4.3) and also
{'fraction_non_alphanumeric': 0.08624923265807244, 'fraction_numerical': 0.007366482504604052, 'mean_word_length': 3.9604261796042617, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4124512', 'n_tokens_mistral': 971, 'n_tokens_neox': 933, 'n_words': 331}