<commit_before>from djangoappengine.db.utils import get_cursor, set_cursor, set_config

from google.appengine.api.datastore import Key

from mapreduce.datastore_range_iterators import AbstractKeyRangeIterator, _KEY_RANGE_ITERATORS
from mapreduce.input_readers import AbstractDatastoreInputReader, _get_params, BadReaderParamsError
from mapreduce import util

class DjangoModelIterator(AbstractKeyRangeIterator):
    def __iter__(self):
        k_range = self._key_range

        # Namespaces are not supported by djangoappengine
        if k_range.namespace:
            return

        model_class = util.for_name(self._query_spec.model_class_path)

        q = model_class.objects.all()

        if k_range.key_start:
            if k_range.include_start:
                q = q.filter(pk__gte=k_range.key_start.id_or_name())
            else:
                q = q.filter(pk__gt=k_range.key_start.id_or_name())

        if k_range.key_end:
            if k_range.include_end:
                q = q.filter(pk__lte=k_range.key_end.id_or_name())
            else:
                q = q.filter(pk__lt=k_range.key_end.id_or_name())

        q = q.order_by('pk')

        q = set_config(q, batch_size=self._query_spec.batch_size)

        if self._cursor:
            q = set_cursor(q, self._cursor)

        self._query = q
<commit_msg>Add query filter support for map-reduce<commit_after>from djangoappengine.db.utils import get_cursor, set_cursor, set_config

from google.appengine.api.datastore import Key

from mapreduce.datastore_range_iterators import AbstractKeyRangeIterator, _KEY_RANGE_ITERATORS
from mapreduce.input_readers import AbstractDatastoreInputReader, _get_params, BadReaderParamsError
from mapreduce import util

class DjangoModelIterator(AbstractKeyRangeIterator):
    def __iter__(self):
        k_range = self._key_range

        # Namespaces are not supported by djangoappengine
        if k_range.namespace:
            return

        model_class = util.for_name(self._query_spec.model_class_path)

        q = model_class.objects.all()

        if k_range.key_start:
            if k_range.include_start:
                q = q.filter(pk__gte=k_range.key_start.id_or_name())
            else:
                q = q.filter(pk__gt=k_range.key_start.id_or_name())

        if k_range.key_end:
            if k_range.include_end:
                q = q.filter(pk__lte=k_range.key_end.id_or_name())
            else:
                q = q.filter(pk__lt=k_range.key_end.id_or_name())

        filters = self._query_spec.filters
        if filters:
            q = q.filter(**dict([[filter[0], filter[2]] for filter in filters]))

        q = q.order_by('pk')

        q = set_config(q, batch_size=self._query_spec.batch_size)

        if self._cursor:
            q = set_cursor(q, self._cursor)

        self._query = q

{'fraction_non_alphanumeric': 0.08177652449770885, 'fraction_numerical': 0.0007049700387733521, 'mean_word_length': 2.6572164948453607, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2128213', 'n_tokens_mistral': 910, 'n_tokens_neox': 883, 'n_words': 164}