<commit_before>from copy import copy
from collections import namedtuple

from stdnet.exceptions import *
from stdnet import pipelines
from stdnet.utils import zip, to_bytestring


queryarg = namedtuple('queryarg','name values unique lookup')
field_query = namedtuple('field_query','query field')

class EmptySet(frozenset):
    query_set = ''
    
    def items(self, slic):
        raise StopIteration
    
    def count(self):
        return len(self)
    

class QuerySet(object):
    '''A QuerySet is not created on its but instead using the model manager.
For example::

    qs = MyModel.objects.filter(field = 'bla)
    
``qs`` is a queryset instance for model ``MyModel``.
'''
    start = None
    stop = None
    lookups = ('in','contains')
    
    def __init__(self, meta, fargs = None, eargs = None,
                 filter_sets = None, ordering = None,
                 queries = None, empty = False):
        '''A query set is  initialized with
        
        * *meta* an model instance meta attribute,
        * *fargs* dictionary containing the lookup parameters to include.
        * *eargs* dictionary containing the lookup parameters to exclude.
        '''
        self._meta  = meta
        self.model  = meta.model            
        self.fargs  = fargs
        self.eargs  = eargs
        self.ordering = ordering
        self.filter_sets = filter_sets
        self.queries = queries
        self.clear()
        if empty:
            self.qset = EmptySet()
        
    def clear(self):
        self.simple = False
        self.qset   = None
        self._seq   = None
        
    def __repr__(self):
        if self._seq is None:
            s = self.__class__.__name__
            if self.fargs:
                s = '%s.filter(%s)' % (s,self.fargs)
            if self.eargs:
                s = '%s.exclude(%s)' % (s,self.eargs)
            return s
        else:
            return str(self._seq)
    
    def __str__(self):
        return self.__repr__()
    
    def __getitem__(self, slic):
        if isinstance(slic,slice):
            return self.aslist(slic)
        return self.aslist()[slic]
    
    def all(self):
        return self
    
    def __iter__(self):
        return self.items()
        
    def filter(self, **kwargs):
        '''Returns a new ``QuerySet`` containing objects that match the\
 given lookup parameters.'''
        if self.fargs:
            c = self.fargs.copy()
            c.update(kwargs)
            kwargs = c
        return self.__class__(self._meta,
                              fargs=kwargs,
                              eargs=self.eargs,
                              filter_sets=self.filter_sets,
                              ordering=self.ordering,
                              queries=self.queries)
    
    def exclude(self, **kwargs):
        '''Returns a new ``QuerySet`` containing objects that do not match\
 the given lookup parameters.'''
        if self.eargs:
            c = self.eargs.copy()
            c.update(kwargs)
            kwargs = c
        return self.__class__(self._meta,
                              fargs=self.fargs,
                              eargs=kwargs,
                              filter_sets=self.filter_sets,
                              ordering=self.ordering,
                              queries=self.queries)
    
    def sort_by(self, ordering):
        '''Sort the query by the given field'''
        if ordering:
            ordering = self._meta.get_sorting(ordering,QuerySetError)
        self.ordering = ordering
        return self
    
    def search(self, text):
        '''Search text in model. A search engine needs to be installed
for this function to be available.'''
        se = self._meta.searchengine
        if se:
            return se.search_model(self.model,text)
        else:
            raise QuerySetError('Search not implemented for {0} model'\
                                .format(self.model))
    
    def get(self):
        items = self.aslist()
        if items:
            if len(items) == 1:
                return items[0]<commit_msg>skip filter and exclude when no data is provided<commit_after>from copy import copy
from collections import namedtuple

from stdnet.exceptions import *
from stdnet import pipelines
from stdnet.utils import zip, to_bytestring


queryarg = namedtuple('queryarg','name values unique lookup')
field_query = namedtuple('field_query','query field')

class EmptySet(frozenset):
    query_set = ''
    
    def items(self, slic):
        raise StopIteration
    
    def count(self):
        return len(self)
    

class QuerySet(object):
    '''A QuerySet is not created on its but instead using the model manager.
For example::

    qs = MyModel.objects.filter(field = 'bla)
    
``qs`` is a queryset instance for model ``MyModel``.
'''
    start = None
    stop = None
    lookups = ('in','contains')
    
    def __init__(self, meta, fargs = None, eargs = None,
                 filter_sets = None, ordering = None,
                 queries = None, empty = False):
        '''A query set is  initialized with
        
        * *meta* an model instance meta attribute,
        * *fargs* dictionary containing the lookup parameters to include.
        * *eargs* dictionary containing the lookup parameters to exclude.
        '''
        self._meta  = meta
        self.model  = meta.model            
        self.fargs  = fargs
        self.eargs  = eargs
        self.ordering = ordering
        self.filter_sets = filter_sets
        self.queries = queries
        self.clear()
        if empty:
            self.qset = EmptySet()
        
    def clear(self):
        self.simple = False
        self.qset   = None
        self._seq   = None
        
    def __repr__(self):
        if self._seq is None:
            s = self.__class__.__name__
            if self.fargs:
                s = '%s.filter(%s)' % (s,self.fargs)
            if self.eargs:
                s = '%s.exclude(%s)' % (s,self.eargs)
            return s
        else:
            return str(self._seq)
    
    def __str__(self):
        return self.__repr__()
    
    def __getitem__(self, slic):
        if isinstance(slic,slice):
            return self.aslist(slic)
        return self.aslist()[slic]
    
    def all(self):
        return self
    
    def __iter__(self):
        return self.items()
        
    def filter(self, **kwargs):
        '''Returns a new ``QuerySet`` containing objects that match the\
 given lookup parameters.'''
        if kwargs:
            if self.fargs:
                c = self.fargs.copy()
                c.update(kwargs)
                kwargs = c
            return self.__class__(self._meta,
                                  fargs=kwargs,
                                  eargs=self.eargs,
                                  filter_sets=self.filter_sets,
                                  ordering=self.ordering,
                                  queries=self.queries)
        else:
            return self
    
    def exclude(self, **kwargs):
        '''Returns a new ``QuerySet`` containing objects that do not match\
 the given lookup parameters.'''
        if kwargs:
            if self.eargs:
                c = self.eargs.copy()
                c.update(kwargs)
                kwargs = c
            return self.__class__(self._meta,
                                  fargs=self.fargs,
                                  eargs=kwargs,
                                  filter_sets=self.filter_sets,
                                  ordering=self.ordering,
                                  queries=self.queries)
        else:
            return self
    
    def sort_by(self, ordering):
        '''Sort the query by the given field'''
        if ordering:
            ordering = self._meta.get_sorting(ordering,QuerySetError)
        self.ordering = ordering
        return self
    
    def search(self, text):
        '''Search text in model. A search engine needs to be installed
for this function to be available.'''
        se = self._meta.searchengine
        if se:
            return se.search_model(self.model,text)
        else:
            raise QuerySetError('Search not implemented for {0} model'\
                                .format(self.model))
    
    def get(self):
        items = self.aslist()
        if items:
            if len(items) == 1:
                return items[0]
{'fraction_non_alphanumeric': 0.08962655601659751, 'fraction_numerical': 0.0007113218731475993, 'mean_word_length': 1.524992517210416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3699950', 'n_tokens_mistral': 2268, 'n_tokens_neox': 2221, 'n_words': 641}