# Path: streamcorpus_pipeline/_exceptions.py
# class ConfigurationError(PipelineBaseException, yakonfig.ConfigurationError):
#     pass
# 
# Path: streamcorpus_pipeline/_spinn3r/protoStream_pb2.py
# class ProtoStreamDelimiter(message.Message):
#   __metaclass__ = reflection.GeneratedProtocolMessageType
#   DESCRIPTOR = _PROTOSTREAMDELIMITER
#   
#   # @@protoc_insertion_point(class_scope:protoStream.ProtoStreamDelimiter)
# 
# class ProtoStreamHeader(message.Message):
#   __metaclass__ = reflection.GeneratedProtocolMessageType
#   DESCRIPTOR = _PROTOSTREAMHEADER
#   
#   # @@protoc_insertion_point(class_scope:protoStream.ProtoStreamHeader)
# 
# Path: streamcorpus_pipeline/stages.py
# class Configured(object):
#     '''Any object containing a configuration.
# 
#     The configuration is passed to the constructor.
# 
#     .. attribute:: config
# 
#         The configuration dictionary for this object.
# 
#     '''
#     def __init__(self, config, *args, **kwargs):
#         super(Configured, self).__init__(*args, **kwargs)
#         self.config = config


import itertools
import json
import logging
import zlib
import streamcorpus
from StringIO import StringIO
from google.protobuf.internal.decoder import _DecodeVarint
from streamcorpus_pipeline._exceptions import ConfigurationError
from streamcorpus_pipeline._spinn3r.protoStream_pb2 \
    import ProtoStreamDelimiter, ProtoStreamHeader
from streamcorpus_pipeline._spinn3r.spinn3rApi_pb2 import Entry
from streamcorpus_pipeline.stages import Configured
        """Read a length from the file, then read that many bytes.
        Return the read block."""
        l = self._read_varint()
        return self._read(l)

    def _read_a(self, cls):
        """Read some protobuf-encoded object stored in a single block
        out of the file."""
        o = cls()
        o.ParseFromString(self._read_block())
        return o

    def _bootstrap(self):
        """Read the header block out of the underlying file.
        After this (and forevermore afterwards) the file will point
        at the start of a ProtoStreamDelimiter block."""
        self._header = self._read_a(ProtoStreamHeader)

    @property
    def header(self):
        if self._header is None:
            self._bootstrap()
        return self._header

    def __iter__(self):
        """Iterate through the objects in the file."""
        if self._header is None:
            self._bootstrap()
        while True:
            delim = self._read_a(ProtoStreamDelimiter)

# Path: streamcorpus_pipeline/_exceptions.py
# class ConfigurationError(PipelineBaseException, yakonfig.ConfigurationError):
#     pass
# 
# Path: streamcorpus_pipeline/_spinn3r/protoStream_pb2.py
# class ProtoStreamDelimiter(message.Message):
#   __metaclass__ = reflection.GeneratedProtocolMessageType
#   DESCRIPTOR = _PROTOSTREAMDELIMITER
#   
#   # @@protoc_insertion_point(class_scope:protoStream.ProtoStreamDelimiter)
# 
# class ProtoStreamHeader(message.Message):
#   __metaclass__ = reflection.GeneratedProtocolMessageType
#   DESCRIPTOR = _PROTOSTREAMHEADER
#   
#   # @@protoc_insertion_point(class_scope:protoStream.ProtoStreamHeader)
# 
# Path: streamcorpus_pipeline/stages.py
# class Configured(object):
#     '''Any object containing a configuration.
# 
#     The configuration is passed to the constructor.
# 
#     .. attribute:: config
# 
#         The configuration dictionary for this object.
# 
#     '''
#     def __init__(self, config, *args, **kwargs):
#         super(Configured, self).__init__(*args, **kwargs)
#         self.config = config


# Path: streamcorpus_pipeline/_spinn3r_feed_storage.py
import itertools
import json
import logging
import zlib
import streamcorpus
from StringIO import StringIO
from google.protobuf.internal.decoder import _DecodeVarint
from streamcorpus_pipeline._exceptions import ConfigurationError
from streamcorpus_pipeline._spinn3r.protoStream_pb2 \
    import ProtoStreamDelimiter, ProtoStreamHeader
from streamcorpus_pipeline._spinn3r.spinn3rApi_pb2 import Entry
from streamcorpus_pipeline.stages import Configured

        """Read a length from the file, then read that many bytes.
        Return the read block."""
        l = self._read_varint()
        return self._read(l)

    def _read_a(self, cls):
        """Read some protobuf-encoded object stored in a single block
        out of the file."""
        o = cls()
        o.ParseFromString(self._read_block())
        return o

    def _bootstrap(self):
        """Read the header block out of the underlying file.
        After this (and forevermore afterwards) the file will point
        at the start of a ProtoStreamDelimiter block."""
        self._header = self._read_a(ProtoStreamHeader)

    @property
    def header(self):
        if self._header is None:
            self._bootstrap()
        return self._header

    def __iter__(self):
        """Iterate through the objects in the file."""
        if self._header is None:
            self._bootstrap()
        while True:
            delim = self._read_a(ProtoStreamDelimiter)

            if delim.delimiter_type == ProtoStreamDelimiter.END: