<commit_before>__all__ = ['DictExtractionTransformer', 'DictListExtractionTransformer']

from . import PureTransformer


class DictExtractionTransformer(PureTransformer):
    """Extract a given key from dictionary object."""

    def __init__(self, key=None, default=None, **kwargs):
        super(DictExtractionTransformer, self).__init__(**kwargs)
        self.set_params(key=key, default=default)
    #end def

    def get_params(self, deep=True):
        params = super(DictExtractionTransformer, self).get_params(deep=deep)
        params['key'] = getattr(self, 'key', None)
        params['default'] = getattr(self, 'default', None)

        return params
    #end def

    def set_params(self, **kwargs):
        super(DictExtractionTransformer, self).set_params(**kwargs)

        self.key = kwargs.pop('key', None)
        self.default = kwargs.pop('default', None)

        return self
    #end def

    def transform_one(self, d): return d.get(self.key, self.default)
#end class


class DictListExtractionTransformer(PureTransformer):
    """Extract a given key from list of dictionary object."""

    def __init__(self, key=None, default=None, **kwargs):
        super(DictListExtractionTransformer, self).__init__(**kwargs)
        self.set_params(key=key, default=default)
    #end def

    def get_params(self, deep=True):
        params = super(DictListExtractionTransformer, self).get_params(deep=deep)
        params['key'] = getattr(self, 'key', None)
        params['default'] = getattr(self, 'default', None)

        return params
    #end def

    def set_params(self, **kwargs):
        super(DictListExtractionTransformer, self).set_params(**kwargs)

        self.key = kwargs.pop('key', None)
        self.default = kwargs.pop('default', None)

        return self
    #end def

    def transform_one(self, L): return [d.get(self.key, self.default) for d in L]
#end class
<commit_msg>Add `FunctionTransformer` and remove unnecessary `get_params` and `set_params` code<commit_after>__all__ = ['DictExtractionTransformer', 'DictListExtractionTransformer', 'FunctionTransformer']

from . import PureTransformer


class DictExtractionTransformer(PureTransformer):
    """Extract a given key from dictionary object."""

    def __init__(self, key=None, default=None, **kwargs):
        super(DictExtractionTransformer, self).__init__(**kwargs)
        self.set_params(key=key, default=default)
    #end def

    def get_params(self, deep=True):
        params = super(DictExtractionTransformer, self).get_params(deep=deep)
        params['key'] = getattr(self, 'key', None)
        params['default'] = getattr(self, 'default', None)

        return params
    #end def

    def set_params(self, **kwargs):
        super(DictExtractionTransformer, self).set_params(**kwargs)

        self.key = kwargs.pop('key', None)
        self.default = kwargs.pop('default', None)

        return self
    #end def

    def transform_one(self, d): return d.get(self.key, self.default)
#end class


class DictListExtractionTransformer(PureTransformer):
    """Extract a given key from list of dictionary object."""

    def __init__(self, key=None, default=None, **kwargs):
        super(DictListExtractionTransformer, self).__init__(**kwargs)

        self.key = key
        self.default = default
    #end def

    def transform_one(self, L): return [d.get(self.key, self.default) for d in L]
#end class


class FunctionTransformer(PureTransformer):
    def __init__(self, func, **kwargs):
        super(FunctionTransformer, self).__init__(**kwargs)
        self.func = func
    #end def

    def transform_one(self, x):
        return self.func(x)
#end class

{'fraction_non_alphanumeric': 0.1223433242506812, 'fraction_numerical': 0.0, 'mean_word_length': 3.6764331210191084, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3484319', 'n_tokens_mistral': 1131, 'n_tokens_neox': 1087, 'n_words': 265}