parameters: (tensors,
             which_bucket,
             batch_size,
             num_buckets,
             num_threads=1,
             capacity=32,
             bucket_capacities=None,
             shapes=None,
             dynamic_pad=False,
             allow_smaller_final_batch=False,
             keep_input=True,
             shared_name=None,
             name=None)
docstring: Lazy bucketing of input tensors according to `which_bucket`.
  
    The argument `tensors` can be a list or a dictionary of tensors.
    The value returned by the function will be of the same type
    as `tensors`.
  
    The tensors entering this function are put into the bucket given by
    `which_bucket`.  Each bucket has its own queue.  When a bucket contains
    `batch_size` elements, this minibatch is pushed onto a top queue.  The
    tensors returned from this function are a the result of dequeueing the
    next minibatch from this top queue.
  
    This function is implemented using several queues. A `QueueRunner` for the
    queues is added to the current `Graph`'s `QUEUE_RUNNER` collection.
  
    As the returned tensors are the result of a dequeue operation, evaluating
    them will throw a `tf.errors.OutOfRangeError` when the input queue is
    exhausted.  If these tensors are feeding another input queue, its queue runner
    will catch this exception, however, if they are used in your main thread
    you are responsible for catching this yourself.
  
    *N.B.:* If `dynamic_pad` is `False`, you must ensure that either
    (i) the `shapes` argument is passed, or (ii) all of the tensors in
    `tensors` must have fully-defined shapes. `ValueError` will be
    raised if neither of these conditions holds.
  
    If `dynamic_pad` is `True`, it is sufficient that the *rank* of the
    tensors is known, but individual dimensions may have shape `None`.
    In this case, for each enqueue the dimensions with value `None`
    may have a variable length; upon dequeue, the output tensors will be padded
    on the right to the maximum shape of the tensors in the current minibatch.
    For numbers, this padding takes value 0.  For strings, this padding is
    the empty string.  See `PaddingFIFOQueue` for more info.
  
    If `allow_smaller_final_batch` is `True`, a smaller batch value than
    `batch_size` is returned when the queues are closed and there are not enough
    elements to fill the batch, otherwise the pending elements are discarded.
    In addition, all output tensors' static shapes, as accessed via the
    `get_shape()` method will have a 0th `Dimension` value of `None`, and
    operations that depend on fixed batch_size would fail.
  
    Args:
      tensors: The list or dictionary of tensors, representing a single element,
        to bucket.  Nested lists are not supported.
      which_bucket: An `int32` scalar Tensor taking a value in `[0, num_buckets)`.
      batch_size: The new batch size pulled from the queue (all queues will have
        the same size).  If a list is passed in then each bucket will have a
        different batch_size.
        (python int, int32 scalar or iterable of integers of length num_buckets).
      num_buckets: A python integer, the number of buckets.
      num_threads: An integer.  The number of threads enqueuing `tensors`.
      capacity: An integer. The maximum number of minibatches in the top queue,
        and also (by default) the maximum number of elements within each bucket.
      bucket_capacities: (Optional) None or a list of integers, the capacities of
        each bucket. If None, capacity is used (default). If specified, it must
        be a list of integers of length num_buckets: the i-th element is used
        as capacity for the i-th bucket queue.
      shapes: (Optional) The shapes for each example.  Defaults to the
        inferred shapes for `tensors`.
      dynamic_pad: Boolean.  Allow variable dimensions in input shapes.
        The given dimensions are padded upon dequeue so that tensors within a
        batch have the same shapes.
      allow_smaller_final_batch: (Optional) Boolean. If `True`, allow the final
        batches to be smaller if there are insufficient items left in the queues.
      keep_input: A `bool` scalar Tensor.  If provided, this tensor controls
        whether the input is added to the queue or not.  If it evaluates `True`,
        then `tensors` are added to the bucket; otherwise they are dropped.  This
        tensor essentially acts as a filtering mechanism.
      shared_name: (Optional). If set, the queues will be shared under the given
        name across multiple sessions.
      name: (Optional) A name for the operations.
  
    Returns:
      A tuple `(bucket, outputs)` where `bucket` is
      a `int32` scalar tensor and `outputs` is a list or
      dictionary of batched outputs corresponding to elements of `tensors`.
      Every step will receive a new bucket of outputs.
  
    Raises:
      ValueError: If the `shapes` are not specified, and cannot be
        inferred from the elements of `tensors` or if batch_size is a sequence
        but its length != num_buckets. Also if bucket_capacities is not None but
        its length != num_buckets.
docstring_summary: Lazy bucketing of input tensors according to `which_bucket`.
docstring_tokens: Lazy bucketing of input tensors according to which_bucket .
function: def bucket(tensors,
             which_bucket,
             batch_size,
             num_buckets,
             num_threads=1,
             capacity=32,
             bucket_capacities=None,
             shapes=None,
             dynamic_pad=False,
             allow_smaller_final_batch=False,
             keep_input=True,
             shared_name=None,
             name=None):
    """Lazy bucketing of input tensors according to `which_bucket`.
  
    The argument `tensors` can be a list or a dictionary of tensors.
    The value returned by the function will be of the same type
    as `tensors`.
  
    The tensors entering this function are put into the bucket given by
    `which_bucket`.  Each bucket has its own queue.  When a bucket contains
    `batch_size` elements, this minibatch is pushed onto a top queue.  The
    tensors returned from this function are a the result of dequeueing the
    next minibatch from this top queue.
  
    This function is implemented using several queues. A `QueueRunner` for the
    queues is added to the current `Graph`'s `QUEUE_RUNNER` collection.
  
    As the returned tensors are the result of a dequeue operation, evaluating
    them will throw a `tf.errors.OutOfRangeError` when the input queue is
    exhausted.  If these tensors are feeding another input queue, its queue runner
    will catch this exception, however, if they are used in your main thread
    you are responsible for catching this yourself.
  
    *N.B.:* If `dynamic_pad` is `False`, you must ensure that either
    (i) the `shapes` argument is passed, or (ii) all of the tensors in
    `tensors` must have fully-defined shapes. `ValueError` will be
    raised if neither of these conditions holds.
  
    If `dynamic_pad` is `True`, it is sufficient that the *rank* of the
    tensors is known, but individual dimensions may have shape `None`.
    In this case, for each enqueue the dimensions with value `None`
    may have a variable length; upon dequeue, the output tensors will be padded
    on the right to the maximum shape of the tensors in the current minibatch.
    For numbers, this padding takes value 0.  For strings, this padding is
    the empty string.  See `PaddingFIFOQueue` for more info.
  
    If `allow_smaller_final_batch` is `True`, a smaller batch value than
    `batch_size` is returned when the queues are closed and there are not enough
    elements to fill the batch, otherwise the pending elements are discarded.
    In addition, all output tensors' static shapes, as accessed via the
    `get_shape()` method will have a 0th `Dimension` value of `None`, and
    operations that depend on fixed batch_size would fail.
  
    Args:
      tensors: The list or dictionary of tensors, representing a single element,
        to bucket.  Nested lists are not supported.
      which_bucket: An `int32` scalar Tensor taking a value in `[0, num_buckets)`.
      batch_size: The new batch size pulled from the queue (all queues will have
        the same size).  If a list is passed in then each bucket will have a
        different batch_size.
        (python int, int32 scalar or iterable of integers of length num_buckets).
      num_buckets: A python integer, the number of buckets.
      num_threads: An integer.  The number of threads enqueuing `tensors`.
      capacity: An integer. The maximum number of minibatches in the top queue,
        and also (by default) the maximum number of elements within each bucket.
      bucket_capacities: (Optional) None or a list of integers, the capacities of
        each bucket. If None, capacity is used (default). If specified, it must
        be a list of integers of length num_buckets: the i-th element is used
        as capacity for the i-th bucket queue.
      shapes: (Optional) The shapes for each example.  Defaults to the
        inferred shapes for `tensors`.
      dynamic_pad: Boolean.  Allow variable dimensions in input shapes.
        The given dimensions are padded upon dequeue so that tensors within a
        batch have the same shapes.
      allow_smaller_final_batch: (Optional) Boolean. If `True`, allow the final
        batches to be smaller if there are insufficient items left in the queues.
      keep_input: A `bool` scalar Tensor.  If provided, this tensor controls
        whether the input is added to the queue or not.  If it evaluates `True`,
        then `tensors` are added to the bucket; otherwise they are dropped.  This
        tensor essentially acts as a filtering mechanism.
      shared_name: (Optional). If set, the queues will be shared under the given
        name across multiple sessions.
      name: (Optional) A name for the operations.
  
    Returns:
      A tuple `(bucket, outputs)` where `bucket` is
      a `int32` scalar tensor and `outputs` is a list or
      dictionary of batched outputs corresponding to elements of `tensors`.
      Every step will receive a new bucket of outputs.
  
    Raises:
      ValueError: If the `shapes` are not specified, and cannot be
        inferred from the elements of `tensors` or if batch_size is a sequence
        but its length != num_buckets. Also if bucket_capacities is not None but
        its length != num_buckets.
    """
    batch_size_per_bucket = False
    if isinstance(batch_size, (list, tuple)):
      batch_size_per_bucket = True
      if len(batch_size) != num_buckets:
        raise ValueError(
            "If batch_size is a list it must have num_buckets elements")
    else:
      batch_size = [batch_size] * num_buckets
  
    if bucket_capacities is None:
      bucket_capacities = [capacity] * num_buckets
    if len(bucket_capacities) != num_buckets:
      raise ValueError(
          "The list bucket_capacities (%s) must have exactly num_buckets (%d) "
          "elements." % (str(bucket_capacities), num_buckets))
  
    tensor_list = _as_tensor_list(tensors)
    with ops.name_scope(name, "bucket", tensor_list) as name:
      tensor_list = _validate_bucket(tensor_list)
      keep_input = _validate_keep_input(keep_input, enqueue_many=False)
      (tensor_list, sparse_info) = _store_sparse_tensors(
          tensor_list, enqueue_many=False, keep_input=keep_input)
  
      # Round-trip batch_size to a tensor, and possibly back
      for i, bucket_batch_size in enumerate(batch_size):
        bucket_batch_size = ops.convert_to_tensor(
            bucket_batch_size, dtype=dtypes.int32, name="batch_size")
        static_batch_size = tensor_util.constant_value(bucket_batch_size)
        batch_size[i] = (static_batch_size if static_batch_size is not None else
                         bucket_batch_size)
  
      types = _dtypes([tensor_list])
      shapes = _shapes([tensor_list], shapes, enqueue_many=False)
  
      which_bucket = ops.convert_to_tensor(
          which_bucket, dtype=dtypes.int32, name="which_bucket")
  
      queue_creator = _which_queue(dynamic_pad)
      bucket_queues = []
      for i in range(num_buckets):
        shared_name_i = ("%s_%d" % (shared_name, i) if shared_name is not None
                         else None)
        bucket_queues.append(
            queue_creator(
                capacity=bucket_capacities[i],
                dtypes=types,
                shapes=shapes,
                shared_name=shared_name_i,
                name="bucket_queue_%d" % i))
  
      maybe_static_batch_size = (
          None if (allow_smaller_final_batch or batch_size_per_bucket)
          else static_batch_size)
  
      bucket_shapes = [
          tensor_shape.vector(maybe_static_batch_size).concatenate(s)
          for s in bucket_queues[0].shapes
      ]
      # top_queue is a PaddingFIFOQueue even if the bucket queues are regular FIFO
      # queues because if we use allow_smaller_final_batch, shapes will
      # contain Nones in their first entry; as a result, a regular
      # FIFOQueue would die when being passed shapes that are not fully defined.
      top_queue = data_flow_ops.PaddingFIFOQueue(
          capacity=capacity,
          dtypes=[dtypes.int32] + types,
          shapes=[tensor_shape.scalar()] + bucket_shapes,
          shared_name=shared_name,
          name="top_queue")
  
      def enqueue_which():
        """Return an op that enqueues conditionally in one of the queues."""
        def enqueue_single(i):
          return bucket_queues[i].enqueue(tensor_list)
  
        enqueues = [
            control_flow_ops.cond(
                math_ops.equal(which_bucket, i),
                functools.partial(enqueue_single, i), control_flow_ops.no_op)
            for i in range(num_buckets)
        ]
        return control_flow_ops.group(*enqueues, name="group_enqueues")
  
      maybe_enqueue = _smart_cond(
          keep_input,
          enqueue_which,
          control_flow_ops.no_op)
  
      bucket_enqueue_ops = [maybe_enqueue] * num_threads
  
      if allow_smaller_final_batch:
        which_dequeue = lambda q: q.dequeue_up_to
      else:
        which_dequeue = lambda q: q.dequeue_many
  
      def make_list(t):
        if isinstance(t, (list, tuple)):
          return t
        else:
          return [t]
  
      enqueues_to_top = [
          top_queue.enqueue(
              [constant_op.constant(i)] + make_list(which_dequeue(q)(
                  bs, name="read_bucket_%d" % i)),
              name="enqueue_from_bucket_%d" % i)
          for i, (q, bs) in enumerate(zip(bucket_queues, batch_size))
      ]
  
      for i, q in enumerate(bucket_queues):
        queue_runner.add_queue_runner(
            queue_runner.QueueRunner(
                q, [enqueues_to_top[i]],
                queue_closed_exception_types=(errors.OutOfRangeError,
                                              errors.CancelledError)))
      queue_runner.add_queue_runner(
          queue_runner.QueueRunner(
              top_queue,
              bucket_enqueue_ops,
              queue_closed_exception_types=(errors.OutOfRangeError,
                                            errors.CancelledError)))
  
      for q in bucket_queues:
        summary.scalar("bucket/%s/size" % q.name,
                       math_ops.cast(top_queue.size(), dtypes.float32))
      summary.scalar("bucket/%s/fraction_of_%d_full" % (top_queue.name, capacity),
                     math_ops.cast(top_queue.size(), dtypes.float32) *
                     (1. / capacity))
  
      dequeued = top_queue.dequeue(name="dequeue_top")
      which_bucket_dequeued = dequeued[0]
      dequeued = dequeued[1:]
      if len(dequeued) == 1:
        dequeued = dequeued[0]
      dequeued = _restore_sparse_tensors(dequeued, sparse_info)
      return (which_bucket_dequeued, _as_original_type(tensors, dequeued))
function_tokens: def bucket ( tensors , which_bucket , batch_size , num_buckets , num_threads = 1 , capacity = 32 , bucket_capacities = None , shapes = None , dynamic_pad = False , allow_smaller_final_batch = False , keep_input = True , shared_name = None , name = None ) : batch_size_per_bucket = False if isinstance ( batch_size , ( list , tuple ) ) : batch_size_per_bucket = True if len ( batch_size ) != num_buckets : raise ValueError ( "If batch_size is a list it must have num_buckets elements" ) else : batch_size = [ batch_size ] * num_buckets if bucket_capacities is None : bucket_capacities = [ capacity ] * num_buckets if len ( bucket_capacities ) != num_buckets : raise ValueError ( "The list bucket_capacities (%s) must have exactly num_buckets (%d) " "elements." % ( str ( bucket_capacities ) , num_buckets ) ) tensor_list = _as_tensor_list ( tensors ) with ops . name_scope ( name , "bucket" , tensor_list ) as name : tensor_list = _validate_bucket ( tensor_list ) keep_input = _validate_keep_input ( keep_input , enqueue_many = False ) ( tensor_list , sparse_info ) = _store_sparse_tensors ( tensor_list , enqueue_many = False , keep_input = keep_input ) # Round-trip batch_size to a tensor, and possibly back for i , bucket_batch_size in enumerate ( batch_size ) : bucket_batch_size = ops . convert_to_tensor ( bucket_batch_size , dtype = dtypes . int32 , name = "batch_size" ) static_batch_size = tensor_util . constant_value ( bucket_batch_size ) batch_size [ i ] = ( static_batch_size if static_batch_size is not None else bucket_batch_size ) types = _dtypes ( [ tensor_list ] ) shapes = _shapes ( [ tensor_list ] , shapes , enqueue_many = False ) which_bucket = ops . convert_to_tensor ( which_bucket , dtype = dtypes . int32 , name = "which_bucket" ) queue_creator = _which_queue ( dynamic_pad ) bucket_queues = [ ] for i in range ( num_buckets ) : shared_name_i = ( "%s_%d" % ( shared_name , i ) if shared_name is not None else None ) bucket_queues . append ( queue_creator ( capacity = bucket_capacities [ i ] , dtypes = types , shapes = shapes , shared_name = shared_name_i , name = "bucket_queue_%d" % i ) ) maybe_static_batch_size = ( None if ( allow_smaller_final_batch or batch_size_per_bucket ) else static_batch_size ) bucket_shapes = [ tensor_shape . vector ( maybe_static_batch_size ) . concatenate ( s ) for s in bucket_queues [ 0 ] . shapes ] # top_queue is a PaddingFIFOQueue even if the bucket queues are regular FIFO # queues because if we use allow_smaller_final_batch, shapes will # contain Nones in their first entry; as a result, a regular # FIFOQueue would die when being passed shapes that are not fully defined. top_queue = data_flow_ops . PaddingFIFOQueue ( capacity = capacity , dtypes = [ dtypes . int32 ] + types , shapes = [ tensor_shape . scalar ( ) ] + bucket_shapes , shared_name = shared_name , name = "top_queue" ) def enqueue_which ( ) : """Return an op that enqueues conditionally in one of the queues.""" def enqueue_single ( i ) : return bucket_queues [ i ] . enqueue ( tensor_list ) enqueues = [ control_flow_ops . cond ( math_ops . equal ( which_bucket , i ) , functools . partial ( enqueue_single , i ) , control_flow_ops . no_op ) for i in range ( num_buckets ) ] return control_flow_ops . group ( * enqueues , name = "group_enqueues" ) maybe_enqueue = _smart_cond ( keep_input , enqueue_which , control_flow_ops . no_op ) bucket_enqueue_ops = [ maybe_enqueue ] * num_threads if allow_smaller_final_batch : which_dequeue = lambda q : q . dequeue_up_to else : which_dequeue = lambda q : q . dequeue_many def make_list ( t ) : if isinstance ( t , ( list , tuple ) ) : return t else : return [ t ] enqueues_to_top = [ top_queue . enqueue ( [ constant_op . constant ( i ) ] + make_list ( which_dequeue ( q ) ( bs , name = "read_bucket_%d" % i ) ) , name = "enqueue_from_bucket_%d" % i ) for i , ( q , bs ) in enumerate ( zip ( bucket_queues , batch_size ) ) ] for i , q in enumerate ( bucket_queues ) : queue_runner . add_queue_runner ( queue_runner . QueueRunner ( q , [ enqueues_to_top [ i ] ] , queue_closed_exception_types = ( errors . OutOfRangeError , errors . CancelledError ) ) ) queue_runner . add_queue_runner ( queue_runner . QueueRunner ( top_queue , bucket_enqueue_ops , queue_closed_exception_types = ( errors . OutOfRangeError , errors . CancelledError ) ) ) for q in bucket_queues : summary . scalar ( "bucket/%s/size" % q . name , math_ops . cast ( top_queue . size ( ) , dtypes . float32 ) ) summary . scalar ( "bucket/%s/fraction_of_%d_full" % ( top_queue . name , capacity ) , math_ops . cast ( top_queue . size ( ) , dtypes . float32 ) * ( 1. / capacity ) ) dequeued = top_queue . dequeue ( name = "dequeue_top" ) which_bucket_dequeued = dequeued [ 0 ] dequeued = dequeued [ 1 : ] if len ( dequeued ) == 1 : dequeued = dequeued [ 0 ] dequeued = _restore_sparse_tensors ( dequeued , sparse_info ) return ( which_bucket_dequeued , _as_original_type ( tensors , dequeued ) )