<commit_before>""" Predefined filters.

"""
from __future__ import absolute_import

from re import compile

__all__ = ("FieldFilter", "RangeFilter", "RegexFilter", "SliceFilter")


class FieldFilter(object):
    """ Filter records by a specific field.
    
    This is intended for use with a Reader or Writer via their filter() method.
    
    """
    def __init__(self, field, values, whitelist=True):
        """ Initialize this object.
        
        By default, records that match one of the given field values are passed
        through and all other records are dropped (whitelisting). If whitelist
        is False this is reversed (blacklisting).
        
        """
        self._field = field
        self._values = set(values)
        self._whitelist = whitelist
        return
        
    def __call__(self, record):
        """ Execute the filter.
        
        """
        try:
            valid = (record[self._field] in self._values) == self._whitelist
        except KeyError:  # no such field
            # The record is invalid for whitelisting because it doesn't have
            # the required field value; for blacklisting it's valid because it
            # doesn't have a prohibited field value.
            valid = not self._whitelist
        return record if valid else None


class RangeFilter(object):
    """ Filter records by a range.
    
    This is intended for use with a Reader or Writer via their filter() method.
    
    """
    def __init__(self, field, min=None, max=None, whitelist=True):
        """ Initialize this object.
        
        By default, records whose 'field' value is within the range [min, max)
        are passed through and all other records are dropped (whitelisting). If
        whitelist is False this is reversed (blacklisting). If 'min' or 'max'
        is None that end of the range is considered to be unlimited. If both
        are None, the filter will pass all records if whitelist is True and
        drop all records if it's False.
                
        """
        self._field = field
        self._min = min
        self._max = max
        self._whitelist = whitelist
        return
        
    def __call__(self, record):
        """ Execute the filter.
        
        """
        value = record[self._field]
        match = ((self._min is None or self._min <= value) and
                 (self._max is None or value < self._max))
        return record if match == self._whitelist else None


class RegexFilter(object):
    """ Filter lines using a regular expression.
    
    This is intended for use with a FilteredIStream or FilteredOStream.
    
    """
    def __init__(self, regex, whitelist=True):
        """ Initialize this object.
        
        By default, lines that match the regular expression are passed through
        and all other lines are dropped (whitelisting). If whitelist is False
        this is reversed (blacklisting).
        
        """
        self._regex = compile(regex)
        self._whitelist = whitelist
        return
        
    def __call__(self, line):
        """ Execute the filter.
        
        """
        valid = (self._regex.search(line) is not None) == self._whitelist
        return line if valid else None


class SliceFilter(object):
    """ Filter lines by slice.
    
    This is intended for use with a FilteredIStream or FilteredOStream.
    
    """
    def __init__(self, expr, values, whitelist=True):
        """ Initialize this object.
        
        The slice expression can be a pair of numbers or a slice object. By
        default, lines where the slice matches one of the values are passed
        through and all other lines are dropped (whitelisting). If whitelist
        is False this is reversed (blacklisting).
        
        """
        try:
            # Create a slice from a [beg, end) pair.
            self._slice = slice(*expr)
        except TypeError:  # not a sequence
            self._slice = expr
        self._values = set(values)
        self._whitelist = whitelist
        return
        
    def __call__(self, line):
        """ Execute the filter.
        
        """
        valid = (line[self._slice] in self._values) == self._whitelist
        return line if valid else None<commit_msg>refactored filter.py module<commit_after>""" Predefined filters.

"""
from __future__ import absolute_import

from re import compile

__all__ = ("FieldFilter", "RangeFilter", "RegexFilter", "SliceFilter")


class FieldFilter(object):
    """ Filter records by a specific field.
    
    This is intended for use with a Reader or Writer via their filter() method.
    
    """
    def __init__(self, field, values, whitelist=True):
        """ Initialize this object.
        
        By default, records that match one of the given field values are passed
        through and all other records are dropped (whitelisting). If whitelist
        is False this is reversed (blacklisting).
        
        """
        self._field = field
        self._values = set(values)
        self._whitelist = whitelist
        return
        
    def __call__(self, record):
        """ Execute the filter.
        
        """
        try:
            match = record[self._field] in self._values
        except KeyError:  # no such field
            # The record is invalid for whitelisting because it doesn't have
            # the required field value; for blacklisting it's valid because it
            # doesn't have a prohibited field value.
            match = False
        return record if match == self._whitelist else None


class RangeFilter(object):
    """ Filter records by a range.
    
    This is intended for use with a Reader or Writer via their filter() method.
    
    """
    def __init__(self, field, min=None, max=None, whitelist=True):
        """ Initialize this object.
        
        By default, records whose 'field' value is within the range [min, max)
        are passed through and all other records are dropped (whitelisting). If
        whitelist is False this is reversed (blacklisting). If 'min' or 'max'
        is None that end of the range is considered to be unlimited. If both
        are None, the filter will pass all records if whitelist is True and
        drop all records if it's False.
                
        """
        self._field = field
        self._min = min
        self._max = max
        self._whitelist = whitelist
        return
        
    def __call__(self, record):
        """ Execute the filter.
        
        """
        value = record[self._field]
        match = ((self._min is None or self._min <= value) and
                 (self._max is None or value < self._max))
        return record if match == self._whitelist else None


class RegexFilter(object):
    """ Filter lines using a regular expression.
    
    This is intended for use with a FilteredIStream or FilteredOStream.
    
    """
    def __init__(self, regex, whitelist=True):
        """ Initialize this object.
        
        By default, lines that match the regular expression are passed through
        and all other lines are dropped (whitelisting). If whitelist is False
        this is reversed (blacklisting).
        
        """
        self._regex = compile(regex)
        self._whitelist = whitelist
        return
        
    def __call__(self, line):
        """ Execute the filter.
        
        """
        match = self._regex.search(line) is not None
        return line if match == self._whitelist else None


class SliceFilter(object):
    """ Filter lines by slice.
    
    This is intended for use with a FilteredIStream or FilteredOStream.
    
    """
    def __init__(self, expr, values, whitelist=True):
        """ Initialize this object.
        
        The slice expression can be a pair of numbers or a slice object. By
        default, lines where the slice matches one of the values are passed
        through and all other lines are dropped (whitelisting). If whitelist
        is False this is reversed (blacklisting).
        
        """
        try:
            # Create a slice from a [beg, end) pair.
            self._slice = slice(*expr)
        except TypeError:  # not a sequence
            self._slice = expr
        self._values = set(values)
        self._whitelist = whitelist
        return
        
    def __call__(self, line):
        """ Execute the filter.
        
        """
        match = line[self._slice] in self._values
        return line if match == self._whitelist else None
{'fraction_non_alphanumeric': 0.0747838279971956, 'fraction_numerical': 0.0, 'mean_word_length': 1.9822299651567945, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2933267', 'n_tokens_mistral': 2214, 'n_tokens_neox': 2145, 'n_words': 898}