# Path: streamcorpus_pipeline/_exceptions.py
# class FailedExtraction(PipelineBaseException):
#     pass
# 
# class ConfigurationError(PipelineBaseException, yakonfig.ConfigurationError):
#     pass
# 
# class FailedVerification(Exception):
#     '''Raised when an md5 verification fails.
#     '''
#     pass


    from cStringIO import StringIO
    from StringIO import StringIO
from functools import partial
from operator import itemgetter
from backports import lzma
    from dossier.fc import FeatureCollection, FeatureCollectionChunk
from streamcorpus_pipeline._exceptions import FailedExtraction, \
    ConfigurationError, FailedVerification
import hashlib
import logging
import os
import os.path as path
import tarfile
import tempfile
import urllib2
import uuid
import pytest
import yakonfig
import streamcorpus
import streamcorpus_pipeline._s3_storage as s3stage
    return os.getenv('AWS_TEST_BUCKET') or 'diffeo-test'


@pytest.fixture(scope='function')
def bucket(request, bucket_name):
    assert os.getenv('AWS_ACCESS_KEY_ID')
    assert os.getenv('AWS_SECRET_ACCESS_KEY')
    return s3stage.get_bucket({'bucket': bucket_name})


def clean_bucket(name, prefix):
    b = s3stage.get_bucket({'bucket': name})
    b.delete_keys(list(b.list(prefix=prefix)))


@pytest.yield_fixture
def from_s3_chunks(bucket_name):
    testc = {
        'from_s3_chunks': {
            'bucket': bucket_name,
            'tries': 1,
            's3_path_prefix': uuid.uuid4().hex,
        },
    }
    defconf = yakonfig.defaulted_config([s3stage.from_s3_chunks], config=testc)
    with defconf as conf:
        yield s3stage.from_s3_chunks(conf['from_s3_chunks'])
        clean_bucket(bucket_name, testc['from_s3_chunks']['s3_path_prefix'])



# Path: streamcorpus_pipeline/_exceptions.py
# class FailedExtraction(PipelineBaseException):
#     pass
# 
# class ConfigurationError(PipelineBaseException, yakonfig.ConfigurationError):
#     pass
# 
# class FailedVerification(Exception):
#     '''Raised when an md5 verification fails.
#     '''
#     pass


# Path: streamcorpus_pipeline/tests/test_s3_storage.py
    from cStringIO import StringIO
    from StringIO import StringIO
from functools import partial
from operator import itemgetter
from backports import lzma
    from dossier.fc import FeatureCollection, FeatureCollectionChunk
from streamcorpus_pipeline._exceptions import FailedExtraction, \
    ConfigurationError, FailedVerification
import hashlib
import logging
import os
import os.path as path
import tarfile
import tempfile
import urllib2
import uuid
import pytest
import yakonfig
import streamcorpus
import streamcorpus_pipeline._s3_storage as s3stage

    return os.getenv('AWS_TEST_BUCKET') or 'diffeo-test'


@pytest.fixture(scope='function')
def bucket(request, bucket_name):
    assert os.getenv('AWS_ACCESS_KEY_ID')
    assert os.getenv('AWS_SECRET_ACCESS_KEY')
    return s3stage.get_bucket({'bucket': bucket_name})


def clean_bucket(name, prefix):
    b = s3stage.get_bucket({'bucket': name})
    b.delete_keys(list(b.list(prefix=prefix)))


@pytest.yield_fixture
def from_s3_chunks(bucket_name):
    testc = {
        'from_s3_chunks': {
            'bucket': bucket_name,
            'tries': 1,
            's3_path_prefix': uuid.uuid4().hex,
        },
    }
    defconf = yakonfig.defaulted_config([s3stage.from_s3_chunks], config=testc)
    with defconf as conf:
        yield s3stage.from_s3_chunks(conf['from_s3_chunks'])
        clean_bucket(bucket_name, testc['from_s3_chunks']['s3_path_prefix'])



@pytest.yield_fixture