def verify_completed_experiment_metadata(experiment_id: int, num_expected_trials: Optional[int]) -> None:
    if (num_expected_trials is not None):
        assert (num_trials(experiment_id) == num_expected_trials)
        assert (num_completed_trials(experiment_id) == num_expected_trials)
    trials = experiment_trials(experiment_id)
    assert (len(trials) > 0)
    for trial in trials:
        if (trial['state'] != 'COMPLETED'):
            report_failed_trial(trial['id'], trial['state'])
            pytest.fail(f"Trial {trial['id']} was not COMPLETED but {trial['state']}")
        assert (len(trial['steps']) > 0)
        batch_ids = [s['total_batches'] for s in trial['steps']]
        assert all(((x <= y) for (x, y) in zip(batch_ids, batch_ids[1:])))
        for step in trial['steps']:
            assert (step['state'] == 'COMPLETED')
            if step['validation']:
                validation = step['validation']
                assert (validation['state'] == 'COMPLETED')
            if step['checkpoint']:
                checkpoint = step['checkpoint']
                assert (checkpoint['state'] in {'COMPLETED', 'DELETED'})
    for trial in trials:
        last_step = trial['steps'][(- 1)]
        assert last_step['checkpoint']
    max_secs_to_free_slots = 30
    for _ in range(max_secs_to_free_slots):
        if (cluster_utils.num_free_slots() == cluster_utils.num_slots()):
            break
        time.sleep(1)
    else:
        raise AssertionError('Slots failed to free after experiment {}'.format(experiment_id))
    run_describe_cli_tests(experiment_id)
    run_list_cli_tests(experiment_id)
def verify_completed_experiment_metadata(experiment_id: int, num_expected_trials: Optional[int]) -> None:
    if (num_expected_trials is not None):
        assert (num_trials(experiment_id) == num_expected_trials)
        assert (num_completed_trials(experiment_id) == num_expected_trials)
    trials = experiment_trials(experiment_id)
    assert (len(trials) > 0)
    for trial in trials:
        if (trial['state'] != 'COMPLETED'):
            report_failed_trial(trial['id'], trial['state'])
            pytest.fail(f"Trial {trial['id']} was not COMPLETED but {trial['state']}")
        assert (len(trial['steps']) > 0)
        batch_ids = [s['total_batches'] for s in trial['steps']]
        assert all(((x <= y) for (x, y) in zip(batch_ids, batch_ids[1:])))
        for step in trial['steps']:
            assert (step['state'] == 'COMPLETED')
            if step['validation']:
                validation = step['validation']
                assert (validation['state'] == 'COMPLETED')
            if step['checkpoint']:
                checkpoint = step['checkpoint']
                assert (checkpoint['state'] in {'COMPLETED', 'DELETED'})
    for trial in trials:
        last_step = trial['steps'][(- 1)]
        assert last_step['checkpoint']
    max_secs_to_free_slots = 30
    for _ in range(max_secs_to_free_slots):
        if (cluster_utils.num_free_slots() == cluster_utils.num_slots()):
            break
        time.sleep(1)
    else:
        raise AssertionError('Slots failed to free after experiment {}'.format(experiment_id))
    run_describe_cli_tests(experiment_id)
    run_list_cli_tests(experiment_id)