<commit_before>from pypika import (
    EmptyCriterion,
    Not,
)
from pypika.functions import Lower


class Filter(object):
    def __init__(self, field_alias, definition):
        self.field_alias = field_alias
        self.definition = definition

    @property
    def is_aggregate(self):
        return self.definition.is_aggregate

    def __eq__(self, other):
        return isinstance(other, self.__class__) \
               and str(self.definition) == str(other.definition)

    def __repr__(self):
        return str(self.definition)


class ComparatorFilter(Filter):
    class Operator(object):
        eq = 'eq'
        ne = 'ne'
        gt = 'gt'
        lt = 'lt'
        gte = 'gte'
        lte = 'lte'

    def __init__(self, field_alias, metric_definition, operator, value):
        definition = getattr(metric_definition, operator)(value)
        super(ComparatorFilter, self).__init__(field_alias, definition)


class BooleanFilter(Filter):
    def __init__(self, field_alias, dimension_definition, value):
        definition = dimension_definition \
            if value \
            else Not(dimension_definition)

        super(BooleanFilter, self).__init__(field_alias, definition)


class ContainsFilter(Filter):
    def __init__(self, field_alias, dimension_definition, values):
        definition = dimension_definition.isin(values)
        super(ContainsFilter, self).__init__(field_alias, definition)


class ExcludesFilter(Filter):
    def __init__(self, field_alias, dimension_definition, values):
        definition = dimension_definition.notin(values)
        super(ExcludesFilter, self).__init__(field_alias, definition)


class RangeFilter(Filter):
    def __init__(self, field_alias, dimension_definition, start, stop):
        definition = dimension_definition[start:stop]
        super(RangeFilter, self).__init__(field_alias, definition)


class PatternFilter(Filter):
    def __init__(self, field_alias, dimension_definition, pattern, *patterns):
        definition = self._apply(dimension_definition, (pattern,) + patterns)
        super(PatternFilter, self).__init__(field_alias, definition)

    def _apply(self, dimension_definition, patterns):
        definition = Lower(dimension_definition).like(Lower(patterns[0]))

        for pattern in patterns[1:]:
            definition |= Lower(dimension_definition).like(Lower(pattern))

        return definition


class AntiPatternFilter(PatternFilter):
    def _apply(self, dimension_definition, pattern):
        return super(AntiPatternFilter, self)._apply(dimension_definition, pattern).negate()


class VoidFilter(Filter):
    def __init__(self, field_alias):
        super(VoidFilter, self).__init__(field_alias, EmptyCriterion())
<commit_msg>Added repr string for VoidFilter<commit_after>from pypika import (
    EmptyCriterion,
    Not,
)
from pypika.functions import Lower


class Filter(object):
    def __init__(self, field_alias, definition):
        self.field_alias = field_alias
        self.definition = definition

    @property
    def is_aggregate(self):
        return self.definition.is_aggregate

    def __eq__(self, other):
        return isinstance(other, self.__class__) \
               and str(self.definition) == str(other.definition)

    def __repr__(self):
        return str(self.definition)


class ComparatorFilter(Filter):
    class Operator(object):
        eq = 'eq'
        ne = 'ne'
        gt = 'gt'
        lt = 'lt'
        gte = 'gte'
        lte = 'lte'

    def __init__(self, field_alias, metric_definition, operator, value):
        definition = getattr(metric_definition, operator)(value)
        super(ComparatorFilter, self).__init__(field_alias, definition)


class BooleanFilter(Filter):
    def __init__(self, field_alias, dimension_definition, value):
        definition = dimension_definition \
            if value \
            else Not(dimension_definition)

        super(BooleanFilter, self).__init__(field_alias, definition)


class ContainsFilter(Filter):
    def __init__(self, field_alias, dimension_definition, values):
        definition = dimension_definition.isin(values)
        super(ContainsFilter, self).__init__(field_alias, definition)


class ExcludesFilter(Filter):
    def __init__(self, field_alias, dimension_definition, values):
        definition = dimension_definition.notin(values)
        super(ExcludesFilter, self).__init__(field_alias, definition)


class RangeFilter(Filter):
    def __init__(self, field_alias, dimension_definition, start, stop):
        definition = dimension_definition[start:stop]
        super(RangeFilter, self).__init__(field_alias, definition)


class PatternFilter(Filter):
    def __init__(self, field_alias, dimension_definition, pattern, *patterns):
        definition = self._apply(dimension_definition, (pattern,) + patterns)
        super(PatternFilter, self).__init__(field_alias, definition)

    def _apply(self, dimension_definition, patterns):
        definition = Lower(dimension_definition).like(Lower(patterns[0]))

        for pattern in patterns[1:]:
            definition |= Lower(dimension_definition).like(Lower(pattern))

        return definition


class AntiPatternFilter(PatternFilter):
    def _apply(self, dimension_definition, pattern):
        return super(AntiPatternFilter, self)._apply(dimension_definition, pattern).negate()


class VoidFilter(Filter):
    def __init__(self, field_alias):
        super(VoidFilter, self).__init__(field_alias, EmptyCriterion())

    def __repr__(self):
        return 'VoidFilter()'

{'fraction_non_alphanumeric': 0.0962922966162707, 'fraction_numerical': 0.0007199424046076314, 'mean_word_length': 3.6776094276094278, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1973836', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1479, 'n_words': 338}