<commit_before>import os
import pytest
import boto3


from gypsy.scripts import DEFAULT_CONF_FILE
from gypsy.utils import _copy_file


DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'data')
BUCKET = os.getenv('GYPSY_BUCKET', '')


@pytest.yield_fixture(scope='module')
def invalid_cli_config_file():
    path = os.path.join(DATA_DIR, 'invalid_cli_config.txt')

    with open(path, 'w') as f:
        f.write('\n\n\n\nnodata\n')

    yield path

    os.remove(path)

@pytest.fixture(scope='session')
def s3_bucket_path():
    return 's3://' + BUCKET

@pytest.fixture(scope='function')
def s3_bucket_conn():
    s3 = boto3.resource('s3')
    bucket_conn = s3.Bucket(BUCKET)

    return bucket_conn

@pytest.yield_fixture(scope='function')
def s3_config_output_dir(s3_bucket_path, s3_bucket_conn):
    prefix = 'test/generate-config/gypsy-output'
    out_dir = '%s/%s' % (s3_bucket_path, prefix)

    yield out_dir

    for key in s3_bucket_conn.objects.filter(Prefix=prefix):
        key.delete()

@pytest.yield_fixture(scope='function')
def s3_prep_output_dir(s3_bucket_path, s3_bucket_conn):
    prefix = 'test/prep/gypsy-output'
    out_dir = '%s/%s' % (s3_bucket_path, prefix)
    files = [
        (os.path.join(DATA_DIR, 'raw_standtable.csv'),
         '/'.join([prefix, 'raw-data.csv'])),
        (DEFAULT_CONF_FILE,
         '/'.join([prefix, 'config.json'])),
    ]

    for item in files:
        _copy_file(item[0], item[1], bucket_conn=s3_bucket_conn)
    data_path = '/'.join([s3_bucket_path, files[0][1]])

    yield {
        'out-dir': out_dir,
        'data-path': data_path
    }

    for key in s3_bucket_conn.objects.filter(Prefix=prefix):
        key.delete()


@pytest.yield_fixture(scope='function')
def s3_simulate_output_dir(s3_bucket_path, s3_bucket_conn):
    # TODO: copy data to s3
    prefix = 'test/simulate/gypsy-output'
    out_dir = '%s/%s' % (s3_bucket_path, prefix)
    files = [
        (os.path.join(DATA_DIR, 'raw_standtable_prepped.csv'),
         '/'.join([prefix, 'raw-data-prepped.csv'])),
        (os.path.join(DATA_DIR, 'raw_standtable.csv'),
         '/'.join([prefix, 'raw-data.csv'])),
        (DEFAULT_CONF_FILE,
         '/'.join([prefix, 'config.json'])),
    ]

    for item in files:
        _copy_file(item[0], item[1], bucket_conn=s3_bucket_conn)

    data_path = '/'.join([s3_bucket_path, files[0][1]])
    prepped_data_path = '/'.join([s3_bucket_path, files[1][1]])

    yield {
        'out-dir': out_dir,
        'data-path': data_path,
        'prepped-data-path': prepped_data_path,
    }

    for key in s3_bucket_conn.objects.filter(Prefix=prefix):
        key.delete()


@pytest.yield_fixture(scope='function')
def config_on_s3(s3_bucket_path, s3_bucket_conn):
    prefix = 'test/_load_and_validate_config/gypsy-ouptput'
    out_dir = '%s/%s' % (s3_bucket_path, prefix)
    files = [
        (DEFAULT_CONF_FILE,
         '/'.join([prefix, 'config.json'])),
    ]

    for item in files:<commit_msg>Skip s3 fixtures if bucket not set<commit_after>import os
import pytest
import boto3


from gypsy.scripts import DEFAULT_CONF_FILE
from gypsy.utils import _copy_file


DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'data')
BUCKET = os.getenv('GYPSY_BUCKET')

SKIP_IF_NO_S3 = pytest.mark.skipif(os.getenv('GYPSY_BUCKET') is None,
                                   reason="S3 tests are not configured locally")

@SKIP_IF_NO_S3
@pytest.yield_fixture(scope='module')
def invalid_cli_config_file():
    path = os.path.join(DATA_DIR, 'invalid_cli_config.txt')

    with open(path, 'w') as f:
        f.write('\n\n\n\nnodata\n')

    yield path

    os.remove(path)

@SKIP_IF_NO_S3
@pytest.fixture(scope='session')
def s3_bucket_path():
    return 's3://' + BUCKET

@SKIP_IF_NO_S3
@pytest.fixture(scope='function')
def s3_bucket_conn():
    s3 = boto3.resource('s3')
    bucket_conn = s3.Bucket(BUCKET)

    return bucket_conn

@SKIP_IF_NO_S3
@pytest.yield_fixture(scope='function')
def s3_config_output_dir(s3_bucket_path, s3_bucket_conn):
    prefix = 'test/generate-config/gypsy-output'
    out_dir = '%s/%s' % (s3_bucket_path, prefix)

    yield out_dir

    for key in s3_bucket_conn.objects.filter(Prefix=prefix):
        key.delete()

@SKIP_IF_NO_S3
@pytest.yield_fixture(scope='function')
def s3_prep_output_dir(s3_bucket_path, s3_bucket_conn):
    prefix = 'test/prep/gypsy-output'
    out_dir = '%s/%s' % (s3_bucket_path, prefix)
    files = [
        (os.path.join(DATA_DIR, 'raw_standtable.csv'),
         '/'.join([prefix, 'raw-data.csv'])),
        (DEFAULT_CONF_FILE,
         '/'.join([prefix, 'config.json'])),
    ]

    for item in files:
        _copy_file(item[0], item[1], bucket_conn=s3_bucket_conn)
    data_path = '/'.join([s3_bucket_path, files[0][1]])

    yield {
        'out-dir': out_dir,
        'data-path': data_path
    }

    for key in s3_bucket_conn.objects.filter(Prefix=prefix):
        key.delete()


@SKIP_IF_NO_S3
@pytest.yield_fixture(scope='function')
def s3_simulate_output_dir(s3_bucket_path, s3_bucket_conn):
    # TODO: copy data to s3
    prefix = 'test/simulate/gypsy-output'
    out_dir = '%s/%s' % (s3_bucket_path, prefix)
    files = [
        (os.path.join(DATA_DIR, 'raw_standtable_prepped.csv'),
         '/'.join([prefix, 'raw-data-prepped.csv'])),
        (os.path.join(DATA_DIR, 'raw_standtable.csv'),
         '/'.join([prefix, 'raw-data.csv'])),
        (DEFAULT_CONF_FILE,
         '/'.join([prefix, 'config.json'])),
    ]

    for item in files:
        _copy_file(item[0], item[1], bucket_conn=s3_bucket_conn)

    data_path = '/'.join([s3_bucket_path, files[0][1]])
    prepped_data_path = '/'.join([s3_bucket_path, files[1][1]])

    yield {
        'out-dir': out_dir,
        'data-path': data_path,
        'prepped-data-path': prepped_data_path,
    }

    for key in s3_bucket_conn.objects.filter(Prefix=prefix):
        key.delete()


@SKIP_IF_NO_S3
@pytest.yield_fixture(scope='function')
def config_on_s3(s3_bucket_path, s3_bucket_conn):
    prefix = 'test/_load_and_validate_config/gypsy-ouptput'
    out_dir = '%s/%s' % (s3_bucket_path, prefix)
    files = [
        (DEFAULT_CONF_FILE,
         '/'.join([prefix, 'config.json'])),
    ]

    for item in files:
{'fraction_non_alphanumeric': 0.15268644747393745, 'fraction_numerical': 0.015396952686447474, 'mean_word_length': 3.422695035460993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3695815', 'n_tokens_mistral': 2402, 'n_tokens_neox': 2379, 'n_words': 370}