# Path: streamcorpus_pipeline/_exceptions.py
# class TransformGivingUp(PipelineBaseException):
#     pass
# 
# class InvalidStreamItem(PipelineBaseException):
#     '''Some content in a stream item was invalid.'''
#     pass
# 
# Path: streamcorpus_pipeline/util.py



import logging
import os
import threading
import time
import uuid
    import gevent
import streamcorpus
from streamcorpus_pipeline._exceptions import TransformGivingUp, \
    InvalidStreamItem
from streamcorpus_pipeline.util import rmtree
        ## operate each transform on this one StreamItem
        for transform in transforms:
            try:
                stream_id = si.stream_id
                si_new = transform(si, context=self.context)

                if si_new is None:
                    logger.warn('transform %r deleted %s abs_url=%r',
                                 transform, stream_id, si and si.abs_url)
                    return None
                si = si_new

            except TransformGivingUp:
                ## do nothing
                logger.info('transform %r giving up on %r',
                            transform, si.stream_id)

            except Exception, exc:
                logger.critical(
                    'transform %r failed on %r from i_str=%r abs_url=%r',
                    transform, si and si.stream_id, self.context.get('i_str'),
                    si and si.abs_url, exc_info=True)

        assert si is not None

        ## expect to always have a stream_time
        if not si.stream_time:
            raise InvalidStreamItem('empty stream_time: %s' % si)

        if si.stream_id is None:

# Path: streamcorpus_pipeline/_exceptions.py
# class TransformGivingUp(PipelineBaseException):
#     pass
# 
# class InvalidStreamItem(PipelineBaseException):
#     '''Some content in a stream item was invalid.'''
#     pass
# 
# Path: streamcorpus_pipeline/util.py



# Path: streamcorpus_pipeline/_pipeline.py
import logging
import os
import threading
import time
import uuid
    import gevent
import streamcorpus
from streamcorpus_pipeline._exceptions import TransformGivingUp, \
    InvalidStreamItem
from streamcorpus_pipeline.util import rmtree

        ## operate each transform on this one StreamItem
        for transform in transforms:
            try:
                stream_id = si.stream_id
                si_new = transform(si, context=self.context)

                if si_new is None:
                    logger.warn('transform %r deleted %s abs_url=%r',
                                 transform, stream_id, si and si.abs_url)
                    return None
                si = si_new

            except TransformGivingUp:
                ## do nothing
                logger.info('transform %r giving up on %r',
                            transform, si.stream_id)

            except Exception, exc:
                logger.critical(
                    'transform %r failed on %r from i_str=%r abs_url=%r',
                    transform, si and si.stream_id, self.context.get('i_str'),
                    si and si.abs_url, exc_info=True)

        assert si is not None

        ## expect to always have a stream_time
        if not si.stream_time:
            raise InvalidStreamItem('empty stream_time: %s' % si)

        if si.stream_id is None:

            raise InvalidStreamItem('empty stream_id: %r' % si)