<commit_before><commit_msg>Added part of the pipeline implementation.<commit_after># Copyright (c) 2013-2014 <NAME>
# Copyright (c) 2013-2014 Other contributers as noted in the CONTRIBUTERS file
#
# This file is part of Phial
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__all__ = ["Pipeline", "TemporaryFile", "concat", "cout"]

# stdlib
import shutil
import StringIO
import sys

class Pipeline(object):
    def prepare_contents(self):
        for i in self.contents:
            assert hasattr(i, "name"), \
                   "{i!r} does not have name attribute.".format(i=i)
            i.seek(0)

    def __init__(self, contents):
        self.contents = contents
        self.prepare_contents()

    def pipe(self, transform):
        self.contents = transform(self.contents)
        self.prepare_contents()
        return self

    def __or__(self, transform):
        return self.pipe(transform)

    def __repr__(self):
        return self.__class__.__name__ + "(" + repr(self.contents) + ")"

class TemporaryFile(StringIO.StringIO):
    def __init__(self, name=None):
        StringIO.StringIO.__init__(self)

        self.name = name

class concat(object):
    def __init__(self, output_name=None):
        self.output_name = output_name

    def __call__(self, contents):
        result = TemporaryFile()
        for i in contents:
            shutil.copyfileobj(i, result)
        return [result]

class cout(object):
	def __init__(self, out=sys.stdout):
		self.out = sys.stdout

    def __call__(self, contents):
        for i in contents:
	        shutil.copyfileobj(i, self.out)
	    return contents
{'fraction_non_alphanumeric': 0.09636277751535191, 'fraction_numerical': 0.009919697685403874, 'mean_word_length': 3.0809248554913293, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3171189', 'n_tokens_mistral': 619, 'n_tokens_neox': 591, 'n_words': 205}