<commit_before>"""
    Saucebrush is a data loading & manipulation framework written in python.
"""

import filters, emitters, sources, utils

class Recipe(filters.Filter):
    
    def __init__(self, *filter_args):
        self._filter_args = filter_args
        self.rejected = []
    
    def get_filters(self):
        filters = []
        for filter_ in self._filter_args:
            if hasattr(filter_, 'get_filters'):
                filters.extend(filter_.get_filters())
            else:
                filters.append(filter_)
        return filters
    
    def run(self, source):
        
        # load filters
        filters = self.get_filters()
        
        # connect datapath
        data = source
        for filter_ in filters:
            data = filter_(self, data)

        # actually run the data through (causes iterators to actually be called)
        for record in data:
            pass

        # try and call done() on all filters
        for filter_ in filters:
            try:
                filter_.done()
            except AttributeError:
                pass    # don't care if there isn't a done method
        
        return self

def run_recipe(source, *filter_args):
    """ Process data, taking it from a source and applying any number of filters
    """
    
    return Recipe(*filter_args).run(source)

<commit_msg>removed stupid chainging stuff. what is this, jQuery?<commit_after>"""
    Saucebrush is a data loading & manipulation framework written in python.
"""

import filters, emitters, sources, utils

class Recipe(filters.Filter):
    
    def __init__(self, *filter_args):
        self._filter_args = filter_args
        self.rejected = []
    
    def get_filters(self):
        
        filters = []
        
        for filter_ in self._filter_args:
            
            # check to see if this is a filter or a recipe
            
            if hasattr(filter_, 'get_filters'):
                # load filters from child recipe
                filters.extend(filter_.get_filters())
                
            else:
                filters.append(filter_)
                
        return filters
    
    def run(self, source):
        
        # load filters
        filters = self.get_filters()
        
        # connect datapath
        data = source
        for filter_ in filters:
            data = filter_(self, data)

        # actually run the data through (causes iterators to actually be called)
        for record in data:
            pass

        # try and call done() on all filters
        for filter_ in filters:
            try:
                filter_.done()
            except AttributeError:
                pass    # don't care if there isn't a done method

def run_recipe(source, *filter_args):
    """ Process data, taking it from a source and applying any number of filters
    """
    
    r = Recipe(*filter_args)
    r.run(source)
    return r


{'fraction_non_alphanumeric': 0.07208432505950357, 'fraction_numerical': 0.0, 'mean_word_length': 1.6843065693430657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7449903', 'n_tokens_mistral': 766, 'n_tokens_neox': 748, 'n_words': 258}