def sum_gradients_all_reduce(dev_prefixes, replica_grads, num_workers, alg, num_shards, gpu_indices):
    'Apply all-reduce algorithm over specified gradient tensors.\n\n  Args:\n    dev_prefixes: list of prefix strings to use to generate PS device names.\n    replica_grads: the gradients to reduce.\n    num_workers: number of worker processes across entire job.\n    alg: the all-reduce algorithm to apply.\n    num_shards: alg-specific sharding factor.\n    gpu_indices: indices of local GPUs in order usable for ring-reduce.\n\n  Returns:\n    list of reduced tensors\n  '
    alg_contains_shuffle = any([(n in alg) for n in ['pscpu', 'psgpu']])
    is_hierarchical = ('/' in alg)
    if ('pscpu' in alg):
        aux_devices = [(prefix + '/cpu:0') for prefix in dev_prefixes]
    elif ('psgpu' in alg):
        aux_devices = [(prefix + ('/gpu:%d' % i)) for i in range(len(gpu_indices)) for prefix in dev_prefixes]
    else:
        aux_devices = ['/job:localhost/cpu:0']
    aux_device_groups = group_device_names(aux_devices, (num_shards if alg_contains_shuffle else 1))
    group_index = 0
    reduced_gv_list = []
    for grad_and_vars in zip(*replica_grads):
        reduced_gv_list.append(sum_grad_and_var_all_reduce(grad_and_vars, num_workers, alg, gpu_indices, (aux_devices if is_hierarchical else aux_device_groups[group_index]), num_shards))
        group_index = ((group_index + 1) % len(aux_device_groups))
    new_replica_grads = [list(x) for x in zip(*reduced_gv_list)]
    return new_replica_grads
Apply all-reduce algorithm over specified gradient tensors.

Args:
  dev_prefixes: list of prefix strings to use to generate PS device names.
  replica_grads: the gradients to reduce.
  num_workers: number of worker processes across entire job.
  alg: the all-reduce algorithm to apply.
  num_shards: alg-specific sharding factor.
  gpu_indices: indices of local GPUs in order usable for ring-reduce.

Returns:
  list of reduced tensors
def sum_gradients_all_reduce(dev_prefixes, replica_grads, num_workers, alg, num_shards, gpu_indices):
    'Apply all-reduce algorithm over specified gradient tensors.\n\n  Args:\n    dev_prefixes: list of prefix strings to use to generate PS device names.\n    replica_grads: the gradients to reduce.\n    num_workers: number of worker processes across entire job.\n    alg: the all-reduce algorithm to apply.\n    num_shards: alg-specific sharding factor.\n    gpu_indices: indices of local GPUs in order usable for ring-reduce.\n\n  Returns:\n    list of reduced tensors\n  '
    alg_contains_shuffle = any([(n in alg) for n in ['pscpu', 'psgpu']])
    is_hierarchical = ('/' in alg)
    if ('pscpu' in alg):
        aux_devices = [(prefix + '/cpu:0') for prefix in dev_prefixes]
    elif ('psgpu' in alg):
        aux_devices = [(prefix + ('/gpu:%d' % i)) for i in range(len(gpu_indices)) for prefix in dev_prefixes]
    else:
        aux_devices = ['/job:localhost/cpu:0']
    aux_device_groups = group_device_names(aux_devices, (num_shards if alg_contains_shuffle else 1))
    group_index = 0
    reduced_gv_list = []
    for grad_and_vars in zip(*replica_grads):
        reduced_gv_list.append(sum_grad_and_var_all_reduce(grad_and_vars, num_workers, alg, gpu_indices, (aux_devices if is_hierarchical else aux_device_groups[group_index]), num_shards))
        group_index = ((group_index + 1) % len(aux_device_groups))
    new_replica_grads = [list(x) for x in zip(*reduced_gv_list)]
    return new_replica_grads