# Path: streamcorpus_pipeline/_exceptions.py
# class FailedExtraction(PipelineBaseException):
#     pass
# 
# class ConfigurationError(PipelineBaseException, yakonfig.ConfigurationError):
#     pass
# 
# class FailedVerification(Exception):
#     '''Raised when an md5 verification fails.
#     '''
#     pass


    from cStringIO import StringIO
    from StringIO import StringIO
from functools import partial
from operator import itemgetter
from backports import lzma
    from dossier.fc import FeatureCollection, FeatureCollectionChunk
from streamcorpus_pipeline._exceptions import FailedExtraction, \
    ConfigurationError, FailedVerification
import hashlib
import logging
import os
import os.path as path
import tarfile
import tempfile
import urllib2
import uuid
import pytest
import yakonfig
import streamcorpus
import streamcorpus_pipeline._s3_storage as s3stage
            if md5:
                # The md5 is on the uncompressed and unencrypted data.
                new_key = md5_key(chunk_key_name, data)
            else:
                new_key = chunk_key_name

            if pre_compress:
                errs, data = streamcorpus.compress_and_encrypt(data)
                assert len(errs) == 0

            chunks[chunk_key_name] = {
                    'key': new_key, 'data': data, 'raw': cdata.getvalue(),
            }
    return chunks


def make_dummy_si_chunks(dummy_chunks, md5=False, pre_compress=True):
    return make_dummy_chunks(dummy_chunks, streamcorpus.Chunk, make_si, 
                             md5=md5, pre_compress=pre_compress)


def make_si(name):
    si = streamcorpus.make_stream_item('2000-01-01T12:34:00.000123Z',
                                       name)
    si.stream_id = name
    si.body.clean_visible = 'clean visible test text'
    return si


def make_dummy_fc_chunks(dummy_chunks, md5=False, pre_compress=True):

# Path: streamcorpus_pipeline/_exceptions.py
# class FailedExtraction(PipelineBaseException):
#     pass
# 
# class ConfigurationError(PipelineBaseException, yakonfig.ConfigurationError):
#     pass
# 
# class FailedVerification(Exception):
#     '''Raised when an md5 verification fails.
#     '''
#     pass


# Path: streamcorpus_pipeline/tests/test_s3_storage.py
    from cStringIO import StringIO
    from StringIO import StringIO
from functools import partial
from operator import itemgetter
from backports import lzma
    from dossier.fc import FeatureCollection, FeatureCollectionChunk
from streamcorpus_pipeline._exceptions import FailedExtraction, \
    ConfigurationError, FailedVerification
import hashlib
import logging
import os
import os.path as path
import tarfile
import tempfile
import urllib2
import uuid
import pytest
import yakonfig
import streamcorpus
import streamcorpus_pipeline._s3_storage as s3stage

            if md5:
                # The md5 is on the uncompressed and unencrypted data.
                new_key = md5_key(chunk_key_name, data)
            else:
                new_key = chunk_key_name

            if pre_compress:
                errs, data = streamcorpus.compress_and_encrypt(data)
                assert len(errs) == 0

            chunks[chunk_key_name] = {
                    'key': new_key, 'data': data, 'raw': cdata.getvalue(),
            }
    return chunks


def make_dummy_si_chunks(dummy_chunks, md5=False, pre_compress=True):
    return make_dummy_chunks(dummy_chunks, streamcorpus.Chunk, make_si, 
                             md5=md5, pre_compress=pre_compress)


def make_si(name):
    si = streamcorpus.make_stream_item('2000-01-01T12:34:00.000123Z',
                                       name)
    si.stream_id = name
    si.body.clean_visible = 'clean visible test text'
    return si


def make_dummy_fc_chunks(dummy_chunks, md5=False, pre_compress=True):

    return make_dummy_chunks(dummy_chunks, FeatureCollectionChunk, make_fc, 