<commit_before>from __future__ import division, absolute_import, print_function

__author__ = 'tamasgal'

from km3pipe.core import Pipeline, Module


class Foo(Module):

    def __init__(self, **context):
        super(self.__class__, self).__init__(**context)
        self.foo = self.get('foo') or 'default_foo'
        self.bar = self.get('bar') or 23

    def process(self, blob):
        print("Processing " + str(self.name))
        print("This is the current blob: " + str(blob))
        print("foo={0}".format(self.foo))
        print("bar={0}".format(self.bar))
        blob['foo_entry'] = self.foo
        return blob

class Moo(Module):
    def process(self, blob):
        print("Processing " + self.name)
        blob['moo_entry'] = 42
        print("This is the blob after processing Moo: " + str(blob))
        return blob


pipe = Pipeline()
pipe.attach(Foo, 'foo_module')
pipe.attach(Moo, 'moo_module')
pipe.drain()<commit_msg>New pump<commit_after>from __future__ import division, absolute_import, print_function

__author__ = 'tamasgal'

from km3pipe.core import Pipeline, Module


class Pump(Module):

    def __init__(self, **context):
        super(self.__class__, self).__init__(**context)
        self.blobs = self.get('blobs') or [{'foo': 1}, {'bar': 2}]

    def process(self, blob):
        try:
            return self.blobs.pop(0)
        except IndexError:
            raise StopIteration


class Foo(Module):

    def __init__(self, **context):
        super(self.__class__, self).__init__(**context)
        self.foo = self.get('foo') or 'default_foo'
        self.bar = self.get('bar') or 23

    def process(self, blob):
        print("This is the current blob: " + str(blob))
        print("foo={0}".format(self.foo))
        print("bar={0}".format(self.bar))
        blob['foo'] = self.foo
        return blob

class Moo(Module):
    def process(self, blob):
        #blob['moo_entry'] = 42
        print("This is the blob after processing Moo: " + str(blob))
        return blob


pipe = Pipeline()
pipe.attach(Pump, 'the_pump')
pipe.attach(Foo, 'foo_module', foo='dummyfoo', bar='dummybar')
pipe.attach(Moo, 'moo_module')
pipe.drain()
{'fraction_non_alphanumeric': 0.14785812989405803, 'fraction_numerical': 0.00783049286043298, 'mean_word_length': 3.082706766917293, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4568552', 'n_tokens_mistral': 718, 'n_tokens_neox': 691, 'n_words': 166}