<commit_before>from tibanna.ec2_utils import (
    UnicornInput,
    Args,
    Config,
    Execution,
    upload_workflow_to_s3,
    get_file_size
)
from tibanna.utils import create_jobid
from tibanna.exceptions import (
    MissingFieldInInputJsonException,
    MalFormattedInputJsonException,
    EC2InstanceLimitException,
    EC2InstanceLimitWaitException
)
import boto3
import pytest


def fun():
    raise Exception("InstanceLimitExceeded")


def test_args():
    input_dict = {'args': {'input_files': {}, 'output_S3_bucket': 'somebucket', 'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args_dict = args.as_dict()
    assert 'input_files' in args_dict
    assert 'app_name' in args_dict
    assert args_dict['app_name'] == 'someapp'


def test_args_missing_field():
    input_dict = {'args': {'input_files': {}, 'app_name': 'someapp'}}
    with pytest.raises(MissingFieldInInputJsonException) as ex:
        Args(**input_dict['args'])
    assert ex
    assert 'output_S3_bucket' in str(ex.value)


def test_args_parse_input_files():
    input_dict = {'args': {'input_files': {"file1": "s3://somebucket/somekey"},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args.fill_default()
    assert hasattr(args, 'input_files')
    assert 'file1' in args.input_files
    assert 'bucket_name' in args.input_files['file1']
    assert 'object_key' in args.input_files['file1']
    assert args.input_files['file1']['bucket_name'] == 'somebucket'
    assert args.input_files['file1']['object_key'] == 'somekey'


def test_args_parse_input_files2():
    input_dict = {'args': {'input_files': {"file1": [["s3://somebucket/somekey1",
                                                      "s3://somebucket/somekey2"],
                                                     ["s3://somebucket/somekey3",
                                                      "s3://somebucket/somekey4"]]},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args.fill_default()
    assert hasattr(args, 'input_files')
    assert 'file1' in args.input_files
    assert 'bucket_name' in args.input_files['file1']
    assert 'object_key' in args.input_files['file1']
    assert args.input_files['file1']['bucket_name'] == 'somebucket'
    assert isinstance(args.input_files['file1']['object_key'], list)
    assert len(args.input_files['file1']['object_key']) == 2
    assert isinstance(args.input_files['file1']['object_key'][0], list)
    assert len(args.input_files['file1']['object_key'][0]) == 2
    assert isinstance(args.input_files['file1']['object_key'][1], list)
    assert len(args.input_files['file1']['object_key'][1]) == 2
    assert args.input_files['file1']['object_key'][0][0] == 'somekey1'
    assert args.input_files['file1']['object_key'][0][1] == 'somekey2'
    assert args.input_files['file1']['object_key'][1][0] == 'somekey3'
    assert args.input_files['file1']['object_key'][1][1] == 'somekey4'


def test_args_parse_input_files3():
    input_dict = {'args': {'input_files': {"file1": ["s3://somebucket/somekey1",
                                                     "s3://somebucket/somekey2"]},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args.fill_default()
    assert hasattr(args, 'input_files')
    assert 'file1' in args.input_files
    assert 'bucket_name' in args.input_files['file1']
    assert 'object_key' in args.input_files['file1']
    assert args.input_files['file1']['bucket_name'] == 'somebucket'
    assert isinstance(args.input_files['file1']['object_key'], list)
    assert len(args.input_files['file1']['object_key']) == 2
    assert args.input_files['file1']['object_key'][0] == 'somekey1'
    assert args.input_files['file1']['object_key'][1] == 'somekey2'


def test_args_parse_input_files_format_error():
    input_dict = {'args': {'input_files': {"file1": "somerandomstr"},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    with pytest.raises(MalFormattedInputJsonException) as ex:
        args.fill_default()
    assert ex
    assert 'S3 url must begin with' in str(ex.value)


def test_args_parse_input_files_format_error2():
    input_dict = {'args': {'input_files': {"file1": ["s3://somebucket/somekey1",
                                                     "s3://otherbucket/somekey2"]},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    with pytest.raises(MalFormattedInputJsonException) as ex:
        args.fill_default()
    assert ex
    assert 'bucket' in str(ex.value)


def test_parse_command():
    input_dict = {'args': {'command': ['command1', 'command2', 'command3'],
                           'output_S3_bucket': 'somebucket',
                           'language': 'shell',
                           'container_image': 'someimage',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args.fill_default()
    assert args.command == 'command1; command2; command3'


def test_config():
    input_dict = {'config': {'log_bucket': 'tibanna-output', 'shutdown_min': 30}}
    cfg = Config(**input_dict['config'])
    cfg_dict = cfg.as_dict()
    assert 'log_bucket' in cfg_dict
    assert 'shutdown_min' in cfg_dict
    assert cfg_dict['shutdown_min'] == 30


def test_config2():
    input_dict = {'config': {'log_bucket': 'tibanna-output'}}
    cfg = Config(**input_dict['config'])
    cfg.fill_default()<commit_msg>more args test<commit_after>from tibanna.ec2_utils import (
    UnicornInput,
    Args,
    Config,
    Execution,
    upload_workflow_to_s3,
    get_file_size
)
from tibanna.utils import create_jobid
from tibanna.exceptions import (
    MissingFieldInInputJsonException,
    MalFormattedInputJsonException,
    EC2InstanceLimitException,
    EC2InstanceLimitWaitException
)
import boto3
import pytest


def fun():
    raise Exception("InstanceLimitExceeded")


def test_args():
    input_dict = {'args': {'input_files': {}, 'output_S3_bucket': 'somebucket', 'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args_dict = args.as_dict()
    assert 'input_files' in args_dict
    assert 'app_name' in args_dict
    assert args_dict['app_name'] == 'someapp'


def test_args_missing_field():
    input_dict = {'args': {'input_files': {}, 'app_name': 'someapp'}}
    with pytest.raises(MissingFieldInInputJsonException) as ex:
        Args(**input_dict['args'])
    assert ex
    assert 'output_S3_bucket' in str(ex.value)


def test_args_parse_input_files():
    input_dict = {'args': {'input_files': {"file1": "s3://somebucket/somekey"},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args.fill_default()
    assert hasattr(args, 'input_files')
    assert 'file1' in args.input_files
    assert 'bucket_name' in args.input_files['file1']
    assert 'object_key' in args.input_files['file1']
    assert args.input_files['file1']['bucket_name'] == 'somebucket'
    assert args.input_files['file1']['object_key'] == 'somekey'


def test_args_parse_input_files2():
    input_dict = {'args': {'input_files': {"file1": [["s3://somebucket/somekey1",
                                                      "s3://somebucket/somekey2"],
                                                     ["s3://somebucket/somekey3",
                                                      "s3://somebucket/somekey4"]]},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args.fill_default()
    assert hasattr(args, 'input_files')
    assert 'file1' in args.input_files
    assert 'bucket_name' in args.input_files['file1']
    assert 'object_key' in args.input_files['file1']
    assert args.input_files['file1']['bucket_name'] == 'somebucket'
    assert isinstance(args.input_files['file1']['object_key'], list)
    assert len(args.input_files['file1']['object_key']) == 2
    assert isinstance(args.input_files['file1']['object_key'][0], list)
    assert len(args.input_files['file1']['object_key'][0]) == 2
    assert isinstance(args.input_files['file1']['object_key'][1], list)
    assert len(args.input_files['file1']['object_key'][1]) == 2
    assert args.input_files['file1']['object_key'][0][0] == 'somekey1'
    assert args.input_files['file1']['object_key'][0][1] == 'somekey2'
    assert args.input_files['file1']['object_key'][1][0] == 'somekey3'
    assert args.input_files['file1']['object_key'][1][1] == 'somekey4'


def test_args_parse_input_files3():
    input_dict = {'args': {'input_files': {"file1": ["s3://somebucket/somekey1",
                                                     "s3://somebucket/somekey2"]},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args.fill_default()
    assert hasattr(args, 'input_files')
    assert 'file1' in args.input_files
    assert 'bucket_name' in args.input_files['file1']
    assert 'object_key' in args.input_files['file1']
    assert args.input_files['file1']['bucket_name'] == 'somebucket'
    assert isinstance(args.input_files['file1']['object_key'], list)
    assert len(args.input_files['file1']['object_key']) == 2
    assert args.input_files['file1']['object_key'][0] == 'somekey1'
    assert args.input_files['file1']['object_key'][1] == 'somekey2'


def test_args_parse_input_files_format_error():
    input_dict = {'args': {'input_files': {"file1": "somerandomstr"},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    with pytest.raises(MalFormattedInputJsonException) as ex:
        args.fill_default()
    assert ex
    assert 'S3 url must begin with' in str(ex.value)


def test_args_parse_input_files_format_error2():
    input_dict = {'args': {'input_files': {"file1": ["s3://somebucket/somekey1",
                                                     "s3://otherbucket/somekey2"]},
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    with pytest.raises(MalFormattedInputJsonException) as ex:
        args.fill_default()
    assert ex
    assert 'bucket' in str(ex.value)


def test_args_input_files_w_mount():
    input_dict = {'args': {'input_files': {
                               "file1": {"bucket_name": "a", "object_key": "b", "mount": True}
                           },
                           'output_S3_bucket': 'somebucket',
                           'cwl_main_filename': 'main.cwl',
                           'cwl_directory_url': 'someurl',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    assert args.input_files['file1']['mount']


def test_parse_command():
    input_dict = {'args': {'command': ['command1', 'command2', 'command3'],
                           'output_S3_bucket': 'somebucket',
                           'language': 'shell',
                           'container_image': 'someimage',
                           'app_name': 'someapp'}}
    args = Args(**input_dict['args'])
    args.fill_default()
    assert args.command == 'command1; command2; command3'


def test_config():
    input_dict = {'config': {'log_bucket': 'tibanna-output', 'shutdown_min': 30}}
    cfg = Config(**input_dict['config'])
    cfg_dict = cfg.as_dict()
    assert 'log_bucket' in cfg_dict
    assert 'shutdown_min' in cfg_dict
    assert cfg_dict['shutdown_min'] == 30


def test_config2():
    input_dict = {'config': {'log_bucket': 'tibanna-output'}}
    cfg = Config(**input_dict['config'])
    cfg.fill_default()
{'fraction_non_alphanumeric': 0.14786312014345487, 'fraction_numerical': 0.01531679617453676, 'mean_word_length': 2.585587998928476, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6257553', 'n_tokens_mistral': 4119, 'n_tokens_neox': 4087, 'n_words': 713}