# Path: streamcorpus_pipeline/stages.py
# class Configured(object):
#     '''Any object containing a configuration.
# 
#     The configuration is passed to the constructor.
# 
#     .. attribute:: config
# 
#         The configuration dictionary for this object.
# 
#     '''
#     def __init__(self, config, *args, **kwargs):
#         super(Configured, self).__init__(*args, **kwargs)
#         self.config = config


from itertools import imap, chain
from urlparse import urlparse
from backports import lzma
from streamcorpus_pipeline.stages import Configured
import logging
import re
    def __call__(self, si, context):
        print si.stream_id, si.abs_url
        return None


class debug_filter(Configured):
    '''Remove all stream items except specified ones.

    This is only needed to debug troublesome configurations if there
    are problems with specific stream items.  It has one
    configuration item:

    .. code-block:: yaml

        accept_stream_ids:
          - 1360783522-ebf059defa41b2812c1f969f28cdb45e

    Only stream IDs in the list are passed on, all others are dropped.
    Defaults to an empty list (drop all stream items).

    '''
    config_name = 'debug_filter'
    default_config = { 'accept_stream_ids': [] }
    def __call__(self, si, context):
        if si.stream_id in self.config['accept_stream_ids']:
            return si
        else:
            return None



# Path: streamcorpus_pipeline/stages.py
# class Configured(object):
#     '''Any object containing a configuration.
# 
#     The configuration is passed to the constructor.
# 
#     .. attribute:: config
# 
#         The configuration dictionary for this object.
# 
#     '''
#     def __init__(self, config, *args, **kwargs):
#         super(Configured, self).__init__(*args, **kwargs)
#         self.config = config


# Path: streamcorpus_pipeline/_filters.py
from itertools import imap, chain
from urlparse import urlparse
from backports import lzma
from streamcorpus_pipeline.stages import Configured
import logging
import re

    def __call__(self, si, context):
        print si.stream_id, si.abs_url
        return None


class debug_filter(Configured):
    '''Remove all stream items except specified ones.

    This is only needed to debug troublesome configurations if there
    are problems with specific stream items.  It has one
    configuration item:

    .. code-block:: yaml

        accept_stream_ids:
          - 1360783522-ebf059defa41b2812c1f969f28cdb45e

    Only stream IDs in the list are passed on, all others are dropped.
    Defaults to an empty list (drop all stream items).

    '''
    config_name = 'debug_filter'
    default_config = { 'accept_stream_ids': [] }
    def __call__(self, si, context):
        if si.stream_id in self.config['accept_stream_ids']:
            return si
        else:
            return None



class id_filter(Configured):