def wait_for_complete(queue_name, job_list=None, job_name_prefix=None, poll_interval=10, idle_log_timeout=None, kill_on_log_timeout=False, stash_log_method=None, tag_instances=False, result_record=None):
    "Return when all jobs in the given list finished.\n\n    If not job list is given, return when all jobs in queue finished.\n\n    Parameters\n    ----------\n    queue_name : str\n        The name of the queue to wait for completion.\n    job_list : Optional[list(dict)]\n        A list of jobID-s in a dict, as returned by the submit function.\n        Example: [{'jobId': 'e6b00f24-a466-4a72-b735-d205e29117b4'}, ...]\n        If not given, this function will return if all jobs completed.\n    job_name_prefix : Optional[str]\n        A prefix for the name of the jobs to wait for. This is useful if the\n        explicit job list is not available but filtering is needed.\n    poll_interval : Optional[int]\n        The time delay between API calls to check the job statuses.\n    idle_log_timeout : Optional[int] or None\n        If not None, then track the logs of the active jobs, and if new output\n        is not produced after `idle_log_timeout` seconds, a warning is printed.\n        If `kill_on_log_timeout` is set to True, the job will also be\n        terminated.\n    kill_on_log_timeout : Optional[bool]\n        If True, and if `idle_log_timeout` is set, jobs will be terminated\n        after timeout. This has no effect if `idle_log_timeout` is None.\n        Default is False.\n    stash_log_method : Optional[str]\n        Select a method to store the job logs, either 's3' or 'local'. If no\n        method is specified, the logs will not be loaded off of AWS. If 's3' is\n        specified, then `job_name_prefix` must also be given, as this will\n        indicate where on s3 to store the logs.\n    tag_instances : bool\n        Default is False. If True, apply tags to the instances. This is toady\n        typically done by each job, so in most cases this should not be needed.\n    result_record : dict\n        A dict which will be modified in place to record the results of the job.\n    "
    if ((stash_log_method == 's3') and (job_name_prefix is None)):
        raise Exception('A job_name_prefix is required to post logs on s3.')
    start_time = datetime.now()
    if (job_list is None):
        job_id_list = []
    else:
        job_id_list = [job['jobId'] for job in job_list]

    def get_jobs_by_status(status, job_id_filter=None, job_name_prefix=None):
        res = batch_client.list_jobs(jobQueue=queue_name, jobStatus=status, maxResults=10000)
        jobs = res['jobSummaryList']
        if job_name_prefix:
            jobs = [job for job in jobs if job['jobName'].startswith(job_name_prefix)]
        if job_id_filter:
            jobs = [job_def for job_def in jobs if (job_def['jobId'] in job_id_filter)]
        return jobs
    job_log_dict = {}

    def check_logs(job_defs):
        'Updates teh job_log_dict.'
        stalled_jobs = set()
        for job_def in job_defs:
            try:
                log_lines = get_job_log(job_def, write_file=False)
                jid = job_def['jobId']
                now = datetime.now()
                if (jid not in job_log_dict.keys()):
                    logger.info(('Adding job %s to the log tracker at %s.' % (jid, now)))
                    job_log_dict[jid] = {'log': log_lines, 'last change time': now}
                elif (len(job_log_dict[jid]['log']) == len(log_lines)):
                    check_dt = (now - job_log_dict[jid]['last change time'])
                    logger.warning(("Job '%s' has not produced output for %d seconds." % (job_def['jobName'], check_dt.seconds)))
                    if (check_dt.seconds > idle_log_timeout):
                        logger.warning(("Job '%s' has stalled." % job_def['jobName']))
                        stalled_jobs.add(jid)
                else:
                    old_log = job_log_dict[jid]['log']
                    old_log += log_lines[len(old_log):]
                    job_log_dict[jid]['last change time'] = now
            except Exception as e:
                logger.error(('Failed to check log for: %s' % str(job_def)))
                logger.exception(e)
        return stalled_jobs
    observed_job_def_dict = {}

    def get_dict_of_job_tuples(job_defs):
        return {jdef['jobId']: [(k, jdef[k]) for k in ['jobName', 'jobId']] for jdef in job_defs}
    batch_client = boto3.client('batch')
    if tag_instances:
        ecs_cluster_name = get_ecs_cluster_for_queue(queue_name, batch_client)
    terminate_msg = 'Job log has stalled for at least %f minutes.'
    terminated_jobs = set()
    stashed_id_set = set()
    while True:
        pre_run = []
        for status in ('SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING'):
            pre_run += get_jobs_by_status(status, job_id_list, job_name_prefix)
        running = get_jobs_by_status('RUNNING', job_id_list, job_name_prefix)
        failed = get_jobs_by_status('FAILED', job_id_list, job_name_prefix)
        done = get_jobs_by_status('SUCCEEDED', job_id_list, job_name_prefix)
        observed_job_def_dict.update(get_dict_of_job_tuples((pre_run + running)))
        logger.info(('(%d s)=(pre: %d, running: %d, failed: %d, done: %d)' % ((datetime.now() - start_time).seconds, len(pre_run), len(running), len(failed), len(done))))
        stalled_jobs = check_logs(running)
        if (idle_log_timeout is not None):
            if kill_on_log_timeout:
                for jid in (stalled_jobs - terminated_jobs):
                    batch_client.terminate_job(jobId=jid, reason=(terminate_msg % (idle_log_timeout / 60.0)))
                    logger.info(('Terminating %s.' % jid))
                    terminated_jobs.add(jid)
        if job_id_list:
            if ((len(failed) + len(done)) == len(job_id_list)):
                ret = 0
                break
        elif (((len(failed) + len(done)) > 0) and ((len(pre_run) + len(running)) == 0)):
            ret = 0
            break
        if tag_instances:
            tag_instances_on_cluster(ecs_cluster_name)
        if stash_log_method:
            stash_logs(observed_job_def_dict, done, failed, queue_name, stash_log_method, job_name_prefix, start_time.strftime('%Y%m%d_%H%M%S'), ids_stashed=stashed_id_set)
        sleep(poll_interval)
    if stash_log_method:
        stash_logs(observed_job_def_dict, done, failed, queue_name, stash_log_method, job_name_prefix, start_time.strftime('%Y%m%d_%H%M%S'), ids_stashed=stashed_id_set)
    result_record['terminated'] = terminated_jobs
    result_record['failed'] = failed
    result_record['succeeded'] = done
    return ret
Return when all jobs in the given list finished.

If not job list is given, return when all jobs in queue finished.

Parameters
----------
queue_name : str
    The name of the queue to wait for completion.
job_list : Optional[list(dict)]
    A list of jobID-s in a dict, as returned by the submit function.
    Example: [{'jobId': 'e6b00f24-a466-4a72-b735-d205e29117b4'}, ...]
    If not given, this function will return if all jobs completed.
job_name_prefix : Optional[str]
    A prefix for the name of the jobs to wait for. This is useful if the
    explicit job list is not available but filtering is needed.
poll_interval : Optional[int]
    The time delay between API calls to check the job statuses.
idle_log_timeout : Optional[int] or None
    If not None, then track the logs of the active jobs, and if new output
    is not produced after `idle_log_timeout` seconds, a warning is printed.
    If `kill_on_log_timeout` is set to True, the job will also be
    terminated.
kill_on_log_timeout : Optional[bool]
    If True, and if `idle_log_timeout` is set, jobs will be terminated
    after timeout. This has no effect if `idle_log_timeout` is None.
    Default is False.
stash_log_method : Optional[str]
    Select a method to store the job logs, either 's3' or 'local'. If no
    method is specified, the logs will not be loaded off of AWS. If 's3' is
    specified, then `job_name_prefix` must also be given, as this will
    indicate where on s3 to store the logs.
tag_instances : bool
    Default is False. If True, apply tags to the instances. This is toady
    typically done by each job, so in most cases this should not be needed.
result_record : dict
    A dict which will be modified in place to record the results of the job.
def wait_for_complete(queue_name, job_list=None, job_name_prefix=None, poll_interval=10, idle_log_timeout=None, kill_on_log_timeout=False, stash_log_method=None, tag_instances=False, result_record=None):
    "Return when all jobs in the given list finished.\n\n    If not job list is given, return when all jobs in queue finished.\n\n    Parameters\n    ----------\n    queue_name : str\n        The name of the queue to wait for completion.\n    job_list : Optional[list(dict)]\n        A list of jobID-s in a dict, as returned by the submit function.\n        Example: [{'jobId': 'e6b00f24-a466-4a72-b735-d205e29117b4'}, ...]\n        If not given, this function will return if all jobs completed.\n    job_name_prefix : Optional[str]\n        A prefix for the name of the jobs to wait for. This is useful if the\n        explicit job list is not available but filtering is needed.\n    poll_interval : Optional[int]\n        The time delay between API calls to check the job statuses.\n    idle_log_timeout : Optional[int] or None\n        If not None, then track the logs of the active jobs, and if new output\n        is not produced after `idle_log_timeout` seconds, a warning is printed.\n        If `kill_on_log_timeout` is set to True, the job will also be\n        terminated.\n    kill_on_log_timeout : Optional[bool]\n        If True, and if `idle_log_timeout` is set, jobs will be terminated\n        after timeout. This has no effect if `idle_log_timeout` is None.\n        Default is False.\n    stash_log_method : Optional[str]\n        Select a method to store the job logs, either 's3' or 'local'. If no\n        method is specified, the logs will not be loaded off of AWS. If 's3' is\n        specified, then `job_name_prefix` must also be given, as this will\n        indicate where on s3 to store the logs.\n    tag_instances : bool\n        Default is False. If True, apply tags to the instances. This is toady\n        typically done by each job, so in most cases this should not be needed.\n    result_record : dict\n        A dict which will be modified in place to record the results of the job.\n    "
    if ((stash_log_method == 's3') and (job_name_prefix is None)):
        raise Exception('A job_name_prefix is required to post logs on s3.')
    start_time = datetime.now()
    if (job_list is None):
        job_id_list = []
    else:
        job_id_list = [job['jobId'] for job in job_list]

    def get_jobs_by_status(status, job_id_filter=None, job_name_prefix=None):
        res = batch_client.list_jobs(jobQueue=queue_name, jobStatus=status, maxResults=10000)
        jobs = res['jobSummaryList']
        if job_name_prefix:
            jobs = [job for job in jobs if job['jobName'].startswith(job_name_prefix)]
        if job_id_filter:
            jobs = [job_def for job_def in jobs if (job_def['jobId'] in job_id_filter)]
        return jobs
    job_log_dict = {}

    def check_logs(job_defs):
        'Updates teh job_log_dict.'
        stalled_jobs = set()
        for job_def in job_defs:
            try:
                log_lines = get_job_log(job_def, write_file=False)
                jid = job_def['jobId']
                now = datetime.now()
                if (jid not in job_log_dict.keys()):
                    logger.info(('Adding job %s to the log tracker at %s.' % (jid, now)))
                    job_log_dict[jid] = {'log': log_lines, 'last change time': now}
                elif (len(job_log_dict[jid]['log']) == len(log_lines)):
                    check_dt = (now - job_log_dict[jid]['last change time'])
                    logger.warning(("Job '%s' has not produced output for %d seconds." % (job_def['jobName'], check_dt.seconds)))
                    if (check_dt.seconds > idle_log_timeout):
                        logger.warning(("Job '%s' has stalled." % job_def['jobName']))
                        stalled_jobs.add(jid)
                else:
                    old_log = job_log_dict[jid]['log']
                    old_log += log_lines[len(old_log):]
                    job_log_dict[jid]['last change time'] = now
            except Exception as e:
                logger.error(('Failed to check log for: %s' % str(job_def)))
                logger.exception(e)
        return stalled_jobs
    observed_job_def_dict = {}

    def get_dict_of_job_tuples(job_defs):
        return {jdef['jobId']: [(k, jdef[k]) for k in ['jobName', 'jobId']] for jdef in job_defs}
    batch_client = boto3.client('batch')
    if tag_instances:
        ecs_cluster_name = get_ecs_cluster_for_queue(queue_name, batch_client)
    terminate_msg = 'Job log has stalled for at least %f minutes.'
    terminated_jobs = set()
    stashed_id_set = set()
    while True:
        pre_run = []
        for status in ('SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING'):
            pre_run += get_jobs_by_status(status, job_id_list, job_name_prefix)
        running = get_jobs_by_status('RUNNING', job_id_list, job_name_prefix)
        failed = get_jobs_by_status('FAILED', job_id_list, job_name_prefix)
        done = get_jobs_by_status('SUCCEEDED', job_id_list, job_name_prefix)
        observed_job_def_dict.update(get_dict_of_job_tuples((pre_run + running)))
        logger.info(('(%d s)=(pre: %d, running: %d, failed: %d, done: %d)' % ((datetime.now() - start_time).seconds, len(pre_run), len(running), len(failed), len(done))))
        stalled_jobs = check_logs(running)
        if (idle_log_timeout is not None):
            if kill_on_log_timeout:
                for jid in (stalled_jobs - terminated_jobs):
                    batch_client.terminate_job(jobId=jid, reason=(terminate_msg % (idle_log_timeout / 60.0)))
                    logger.info(('Terminating %s.' % jid))
                    terminated_jobs.add(jid)
        if job_id_list:
            if ((len(failed) + len(done)) == len(job_id_list)):
                ret = 0
                break
        elif (((len(failed) + len(done)) > 0) and ((len(pre_run) + len(running)) == 0)):
            ret = 0
            break
        if tag_instances:
            tag_instances_on_cluster(ecs_cluster_name)
        if stash_log_method:
            stash_logs(observed_job_def_dict, done, failed, queue_name, stash_log_method, job_name_prefix, start_time.strftime('%Y%m%d_%H%M%S'), ids_stashed=stashed_id_set)
        sleep(poll_interval)
    if stash_log_method:
        stash_logs(observed_job_def_dict, done, failed, queue_name, stash_log_method, job_name_prefix, start_time.strftime('%Y%m%d_%H%M%S'), ids_stashed=stashed_id_set)
    result_record['terminated'] = terminated_jobs
    result_record['failed'] = failed
    result_record['succeeded'] = done
    return ret