# Path: streamcorpus_pipeline/_exceptions.py
# class FailedExtraction(PipelineBaseException):
#     pass
# 
# class ConfigurationError(PipelineBaseException, yakonfig.ConfigurationError):
#     pass
# 
# class FailedVerification(Exception):
#     '''Raised when an md5 verification fails.
#     '''
#     pass


    from cStringIO import StringIO
    from StringIO import StringIO
from functools import partial
from operator import itemgetter
from backports import lzma
    from dossier.fc import FeatureCollection, FeatureCollectionChunk
from streamcorpus_pipeline._exceptions import FailedExtraction, \
    ConfigurationError, FailedVerification
import hashlib
import logging
import os
import os.path as path
import tarfile
import tempfile
import urllib2
import uuid
import pytest
import yakonfig
import streamcorpus
import streamcorpus_pipeline._s3_storage as s3stage
    from_s3_chunks.config['input_format'] = chunker['format']
    from_s3_chunks.config['compare_md5_in_file_name'] = True

    the_key = dummies.keys()[0]
    # Corrupt the data so the md5 won't match.
    fullkey = path.join(from_s3_chunks.config['s3_path_prefix'],
                        chunks[the_key]['key'])
    key = bucket.get_key(fullkey)
    key.set_contents_from_string(lzma.compress('FUBAR'))

    with pytest.raises(FailedVerification):
        from_s3_chunks(chunks[the_key]['key'])


def test_from_s3_chunks_bad_informat(bucket, from_s3_chunks, chunker):
    dummies = {'a': ['a1']}
    from_s3_chunks.config['input_format'] = chunker['format']
    put_chunks(bucket, chunker['make_chunks'](dummies),
               prefix=from_s3_chunks.config['s3_path_prefix'])

    from_s3_chunks.config['input_format'] = 'FUBAR'
    with pytest.raises(ConfigurationError):
        from_s3_chunks('a')


def test_to_s3_chunks_bad_outformat(bucket, to_s3_chunks, chunker):
    dummies = {'a': ['a1']}
    chunks = chunker['make_chunks'](dummies)

    to_s3_chunks.config['output_format'] = 'FUBAR'

# Path: streamcorpus_pipeline/_exceptions.py
# class FailedExtraction(PipelineBaseException):
#     pass
# 
# class ConfigurationError(PipelineBaseException, yakonfig.ConfigurationError):
#     pass
# 
# class FailedVerification(Exception):
#     '''Raised when an md5 verification fails.
#     '''
#     pass


# Path: streamcorpus_pipeline/tests/test_s3_storage.py
    from cStringIO import StringIO
    from StringIO import StringIO
from functools import partial
from operator import itemgetter
from backports import lzma
    from dossier.fc import FeatureCollection, FeatureCollectionChunk
from streamcorpus_pipeline._exceptions import FailedExtraction, \
    ConfigurationError, FailedVerification
import hashlib
import logging
import os
import os.path as path
import tarfile
import tempfile
import urllib2
import uuid
import pytest
import yakonfig
import streamcorpus
import streamcorpus_pipeline._s3_storage as s3stage

    from_s3_chunks.config['input_format'] = chunker['format']
    from_s3_chunks.config['compare_md5_in_file_name'] = True

    the_key = dummies.keys()[0]
    # Corrupt the data so the md5 won't match.
    fullkey = path.join(from_s3_chunks.config['s3_path_prefix'],
                        chunks[the_key]['key'])
    key = bucket.get_key(fullkey)
    key.set_contents_from_string(lzma.compress('FUBAR'))

    with pytest.raises(FailedVerification):
        from_s3_chunks(chunks[the_key]['key'])


def test_from_s3_chunks_bad_informat(bucket, from_s3_chunks, chunker):
    dummies = {'a': ['a1']}
    from_s3_chunks.config['input_format'] = chunker['format']
    put_chunks(bucket, chunker['make_chunks'](dummies),
               prefix=from_s3_chunks.config['s3_path_prefix'])

    from_s3_chunks.config['input_format'] = 'FUBAR'
    with pytest.raises(ConfigurationError):
        from_s3_chunks('a')


def test_to_s3_chunks_bad_outformat(bucket, to_s3_chunks, chunker):
    dummies = {'a': ['a1']}
    chunks = chunker['make_chunks'](dummies)

    to_s3_chunks.config['output_format'] = 'FUBAR'

    with pytest.raises(ConfigurationError):