<commit_before>import page
import six
from six import text_type
from mwclient.util import parse_timestamp


class List(object):

    def __init__(self, site, list_name, prefix, limit=None, return_values=None, max_items=None, *args, **kwargs):
        # NOTE: Fix limit
        self.site = site
        self.list_name = list_name
        self.generator = 'list'
        self.prefix = prefix

        kwargs.update(args)
        self.args = kwargs

        if limit is None:
            limit = site.api_limit
        self.args[self.prefix + 'limit'] = text_type(limit)

        self.count = 0
        self.max_items = max_items

        self._iter = iter(xrange(0))

        self.last = False
        self.result_member = list_name
        self.return_values = return_values

    def __iter__(self):
        return self

    def __next__(self, full=False):
        if self.max_items is not None:
            if self.count >= self.max_items:
                raise StopIteration
        try:
            item = six.next(self._iter)
            self.count += 1
            if 'timestamp' in item:
                item['timestamp'] = parse_timestamp(item['timestamp'])
            if full:
                return item

            if type(self.return_values) is tuple:
                return tuple((item[i] for i in self.return_values))
            elif self.return_values is None:
                return item
            else:
                return item[self.return_values]

        except StopIteration:
            if self.last:
                raise StopIteration
            self.load_chunk()
            return List.next(self, full=full)

    def next(self, full=False):
        """ For Python 2.x support """
        return self.__next__(full)

    def load_chunk(self):
        data = self.site.api('query', (self.generator, self.list_name), *[(text_type(k), v) for k, v in six.iteritems(self.args)])
        if not data:
            # Non existent page
            raise StopIteration
        self.set_iter(data)

        if data.get('continue'):
            # New style continuation, added in MediaWiki 1.21
            self.args.update(data['continue'])

        elif self.list_name in data.get('query-continue', ()):
            # Old style continuation
            self.args.update(data['query-continue'][self.list_name])

        else:
            self.last = True

    def set_iter(self, data):
        if self.result_member not in data['query']:
            self._iter = iter(xrange(0))
        elif type(data['query'][self.result_member]) is list:
            self._iter = iter(data['query'][self.result_member])
        else:
            self._iter = six.itervalues(data['query'][self.result_member])

    def __repr__(self):
        return "<List object '%s' for %s>" % (self.list_name, self.site)

    @staticmethod
    def generate_kwargs(_prefix, *args, **kwargs):
        kwargs.update(args)
        for key, value in six.iteritems(kwargs):
            if value is not None and value is not False:
                yield _prefix + key, value

    @staticmethod
    def get_prefix(prefix, generator=False):
        if generator:
            return 'g' + prefix
        else:
            return prefix
<commit_msg>[#52] Python 3 compat: Use six.range<commit_after>import page
import six
import six.moves
from six import text_type
from mwclient.util import parse_timestamp


class List(object):

    def __init__(self, site, list_name, prefix, limit=None, return_values=None, max_items=None, *args, **kwargs):
        # NOTE: Fix limit
        self.site = site
        self.list_name = list_name
        self.generator = 'list'
        self.prefix = prefix

        kwargs.update(args)
        self.args = kwargs

        if limit is None:
            limit = site.api_limit
        self.args[self.prefix + 'limit'] = text_type(limit)

        self.count = 0
        self.max_items = max_items

        self._iter = iter(six.moves.range(0))

        self.last = False
        self.result_member = list_name
        self.return_values = return_values

    def __iter__(self):
        return self

    def __next__(self, full=False):
        if self.max_items is not None:
            if self.count >= self.max_items:
                raise StopIteration
        try:
            item = six.next(self._iter)
            self.count += 1
            if 'timestamp' in item:
                item['timestamp'] = parse_timestamp(item['timestamp'])
            if full:
                return item

            if type(self.return_values) is tuple:
                return tuple((item[i] for i in self.return_values))
            elif self.return_values is None:
                return item
            else:
                return item[self.return_values]

        except StopIteration:
            if self.last:
                raise StopIteration
            self.load_chunk()
            return List.next(self, full=full)

    def next(self, full=False):
        """ For Python 2.x support """
        return self.__next__(full)

    def load_chunk(self):
        data = self.site.api('query', (self.generator, self.list_name), *[(text_type(k), v) for k, v in six.iteritems(self.args)])
        if not data:
            # Non existent page
            raise StopIteration
        self.set_iter(data)

        if data.get('continue'):
            # New style continuation, added in MediaWiki 1.21
            self.args.update(data['continue'])

        elif self.list_name in data.get('query-continue', ()):
            # Old style continuation
            self.args.update(data['query-continue'][self.list_name])

        else:
            self.last = True

    def set_iter(self, data):
        if self.result_member not in data['query']:
            self._iter = iter(six.moves.range(0))
        elif type(data['query'][self.result_member]) is list:
            self._iter = iter(data['query'][self.result_member])
        else:
            self._iter = six.itervalues(data['query'][self.result_member])

    def __repr__(self):
        return "<List object '%s' for %s>" % (self.list_name, self.site)

    @staticmethod
    def generate_kwargs(_prefix, *args, **kwargs):
        kwargs.update(args)
        for key, value in six.iteritems(kwargs):
            if value is not None and value is not False:
                yield _prefix + key, value

    @staticmethod
    def get_prefix(prefix, generator=False):
        if generator:
            return 'g' + prefix
        else:
            return prefix

{'fraction_non_alphanumeric': 0.1024854249769868, 'fraction_numerical': 0.002915004602638846, 'mean_word_length': 1.9285714285714286, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '724916', 'n_tokens_mistral': 1878, 'n_tokens_neox': 1814, 'n_words': 483}