@mark.parametrize('with_commandline', (True, False))
def test_nevergrad_example(with_commandline: bool, tmpdir: Path) -> None:
    budget = (32 if with_commandline else 1)
    cmd = ['example/my_app.py', '-m', ('hydra.sweep.dir=' + str(tmpdir)), f'hydra.sweeper.optim.budget={budget}', f'hydra.sweeper.optim.num_workers={min(8, budget)}', 'hydra.sweeper.optim.seed=12']
    if with_commandline:
        cmd += ['db=mnist,cifar', 'batch_size=4,8,12,16', 'lr=tag(log, interval(0.001, 1.0))', 'dropout=interval(0,1)']
    run_python_script(cmd)
    returns = OmegaConf.load(f'{tmpdir}/optimization_results.yaml')
    assert isinstance(returns, DictConfig)
    assert (returns.name == 'nevergrad')
    assert (len(returns) == 3)
    best_parameters = returns.best_evaluated_params
    assert (not best_parameters.dropout.is_integer())
    if (budget > 1):
        assert (best_parameters.batch_size == 4)
    last_job = max((int(fp.name) for fp in Path(tmpdir).iterdir() if fp.name.isdigit()))
    assert (last_job == (budget - 1))
@mark.parametrize('with_commandline', (True, False))
def test_nevergrad_example(with_commandline: bool, tmpdir: Path) -> None:
    budget = (32 if with_commandline else 1)
    cmd = ['example/my_app.py', '-m', ('hydra.sweep.dir=' + str(tmpdir)), f'hydra.sweeper.optim.budget={budget}', f'hydra.sweeper.optim.num_workers={min(8, budget)}', 'hydra.sweeper.optim.seed=12']
    if with_commandline:
        cmd += ['db=mnist,cifar', 'batch_size=4,8,12,16', 'lr=tag(log, interval(0.001, 1.0))', 'dropout=interval(0,1)']
    run_python_script(cmd)
    returns = OmegaConf.load(f'{tmpdir}/optimization_results.yaml')
    assert isinstance(returns, DictConfig)
    assert (returns.name == 'nevergrad')
    assert (len(returns) == 3)
    best_parameters = returns.best_evaluated_params
    assert (not best_parameters.dropout.is_integer())
    if (budget > 1):
        assert (best_parameters.batch_size == 4)
    last_job = max((int(fp.name) for fp in Path(tmpdir).iterdir() if fp.name.isdigit()))
    assert (last_job == (budget - 1))