<commit_before>#
# This file is a part of the normalize python library
#
# normalize is free software: you can redistribute it and/or modify
# it under the terms of the MIT License.
#
# normalize is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# MIT License for more details.
#
# You should have received a copy of the MIT license along with
# normalize.  If not, refer to the upstream repository at
# http://github.com/hearsaycorp/normalize
#

from __future__ import absolute_import

from normalize.coll import Collection
import normalize.exc as exc
from normalize.record import Record
from normalize.selector import FieldSelector


class Visitor(object):
    """The Visitor object represents a single recursive visit in progress.  You
    hopefully shouldn't have to sub-class this class for most use cases; just
    VisitorPattern.
    """
    def __init__(self, unpack_func, apply_func, collect_func, reduce_func,
                 apply_empty_slots=False, extraneous=False,
                 ignore_empty_string=False, ignore_none=True,
                 visit_filter=None):
        """Create a new Visitor object.  Generally called by a front-end class
        method of :py:class:`VisitorPattern`

        There are four positional arguments, which specify the particular
        functions to be used during the visit.  The important options from a
        user of a visitor are the keyword arguments:

            ``apply_empty_slots=``\ *bool*
                If set, then your ``apply`` method (or ``reverse``, etc) will
                be called even if there is no corresponding value in the input.
                Your method will receive the Exception as if it were the value.

            ``extraneous=``\ *bool*
                Also call the apply method on properties marked *extraneous*.
                False by default.

            ``ignore_empty_string=``\ *bool*
                If the 'apply' function returns the empty string, treat it as
                if the slot or object did not exist.  ``False`` by default.

            ``ignore_none=``\ *bool*
                If the 'apply' function returns ``None``, treat it as if the
                slot or object did not exist.  ``True`` by default.

            ``visit_filter=``\ *MultiFieldSelector*
                This supplies an instance of
                :py:class:`normalize.selector.MultiFieldSelector`, and
                restricts the operation to the matched object fields.
        """
        self.unpack = unpack_func
        self.apply = apply_func
        self.collect = collect_func
        self.reduce = reduce_func

        self.apply_empty_slots = apply_empty_slots
        self.extraneous = extraneous
        self.ignore_empty_string = ignore_empty_string
        self.ignore_none = ignore_none

        self.visit_filter = visit_filter

        self.seen = set()  # TODO
        self.cue = list()

    def is_filtered(self, prop):<commit_msg>Visitor: accept 'filter' as well as 'visit_filter'<commit_after>#
# This file is a part of the normalize python library
#
# normalize is free software: you can redistribute it and/or modify
# it under the terms of the MIT License.
#
# normalize is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# MIT License for more details.
#
# You should have received a copy of the MIT license along with
# normalize.  If not, refer to the upstream repository at
# http://github.com/hearsaycorp/normalize
#

from __future__ import absolute_import

from normalize.coll import Collection
import normalize.exc as exc
from normalize.record import Record
from normalize.selector import FieldSelector


class Visitor(object):
    """The Visitor object represents a single recursive visit in progress.  You
    hopefully shouldn't have to sub-class this class for most use cases; just
    VisitorPattern.
    """
    def __init__(self, unpack_func, apply_func, collect_func, reduce_func,
                 apply_empty_slots=False, extraneous=False,
                 ignore_empty_string=False, ignore_none=True,
                 visit_filter=None, filter=None):
        """Create a new Visitor object.  Generally called by a front-end class
        method of :py:class:`VisitorPattern`

        There are four positional arguments, which specify the particular
        functions to be used during the visit.  The important options from a
        user of a visitor are the keyword arguments:

            ``apply_empty_slots=``\ *bool*
                If set, then your ``apply`` method (or ``reverse``, etc) will
                be called even if there is no corresponding value in the input.
                Your method will receive the Exception as if it were the value.

            ``extraneous=``\ *bool*
                Also call the apply method on properties marked *extraneous*.
                False by default.

            ``ignore_empty_string=``\ *bool*
                If the 'apply' function returns the empty string, treat it as
                if the slot or object did not exist.  ``False`` by default.

            ``ignore_none=``\ *bool*
                If the 'apply' function returns ``None``, treat it as if the
                slot or object did not exist.  ``True`` by default.

            ``visit_filter=``\ *MultiFieldSelector*
                This supplies an instance of
                :py:class:`normalize.selector.MultiFieldSelector`, and
                restricts the operation to the matched object fields.
        """
        self.unpack = unpack_func
        self.apply = apply_func
        self.collect = collect_func
        self.reduce = reduce_func

        self.apply_empty_slots = apply_empty_slots
        self.extraneous = extraneous
        self.ignore_empty_string = ignore_empty_string
        self.ignore_none = ignore_none

        if visit_filter is None:
            visit_filter = filter
        self.visit_filter = visit_filter

        self.seen = set()  # TODO
        self.cue = list()

    def is_filtered(self, prop):
{'fraction_non_alphanumeric': 0.06664535719993607, 'fraction_numerical': 0.0, 'mean_word_length': 2.470881863560732, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7420552', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1531, 'n_words': 662}