parameters: (file_pattern,
                                batch_size,
                                features,
                                reader,
                                randomize_input=True,
                                num_epochs=None,
                                queue_capacity=10000,
                                reader_num_threads=1,
                                feature_queue_capacity=100,
                                num_queue_runners=2,
                                parser_num_threads=None,
                                name=None)
docstring: Adds operations to read, queue, batch and parse `Example` protos.
  
    Given file pattern (or list of files), will setup a queue for file names,
    read `Example` proto using provided `reader`, use batch queue to create
    batches of examples of size `batch_size` and parse example given `features`
    specification.
  
    All queue runners are added to the queue runners collection, and may be
    started via `start_queue_runners`.
  
    All ops are added to the default graph.
  
    Args:
      file_pattern: List of files or pattern of file paths containing
          `Example` records. See `tf.gfile.Glob` for pattern rules.
      batch_size: An int or scalar `Tensor` specifying the batch size to use.
      features: A `dict` mapping feature keys to `FixedLenFeature` or
        `VarLenFeature` values.
      reader: A function or class that returns an object with
        `read` method, (filename tensor) -> (example tensor).
      randomize_input: Whether the input should be randomized.
      num_epochs: Integer specifying the number of times to read through the
        dataset. If None, cycles through the dataset forever. NOTE - If specified,
        creates a variable that must be initialized, so call
        tf.initialize_local_variables() as shown in the tests.
      queue_capacity: Capacity for input queue.
      reader_num_threads: The number of threads to read examples.
      feature_queue_capacity: Capacity of the parsed features queue.
      num_queue_runners: Number of queue runners to start for the feature queue,
        Adding multiple queue runners for the parsed example queue helps maintain
        a full queue when the subsequent computations overall are cheaper than
        parsing.
      parser_num_threads: (Deprecated) The number of threads to parse examples.
      name: Name of resulting op.
  
    Returns:
      A dict of `Tensor` or `SparseTensor` objects for each in `features`.
      If `keep_keys` is `True`, returns tuple of string `Tensor` and above dict.
  
    Raises:
      ValueError: for invalid inputs.
docstring_summary: Adds operations to read, queue, batch and parse `Example` protos.
docstring_tokens: Adds operations to read queue batch and parse Example protos .
function: def read_keyed_batch_features(file_pattern,
                                batch_size,
                                features,
                                reader,
                                randomize_input=True,
                                num_epochs=None,
                                queue_capacity=10000,
                                reader_num_threads=1,
                                feature_queue_capacity=100,
                                num_queue_runners=2,
                                parser_num_threads=None,
                                name=None):
    """Adds operations to read, queue, batch and parse `Example` protos.
  
    Given file pattern (or list of files), will setup a queue for file names,
    read `Example` proto using provided `reader`, use batch queue to create
    batches of examples of size `batch_size` and parse example given `features`
    specification.
  
    All queue runners are added to the queue runners collection, and may be
    started via `start_queue_runners`.
  
    All ops are added to the default graph.
  
    Args:
      file_pattern: List of files or pattern of file paths containing
          `Example` records. See `tf.gfile.Glob` for pattern rules.
      batch_size: An int or scalar `Tensor` specifying the batch size to use.
      features: A `dict` mapping feature keys to `FixedLenFeature` or
        `VarLenFeature` values.
      reader: A function or class that returns an object with
        `read` method, (filename tensor) -> (example tensor).
      randomize_input: Whether the input should be randomized.
      num_epochs: Integer specifying the number of times to read through the
        dataset. If None, cycles through the dataset forever. NOTE - If specified,
        creates a variable that must be initialized, so call
        tf.initialize_local_variables() as shown in the tests.
      queue_capacity: Capacity for input queue.
      reader_num_threads: The number of threads to read examples.
      feature_queue_capacity: Capacity of the parsed features queue.
      num_queue_runners: Number of queue runners to start for the feature queue,
        Adding multiple queue runners for the parsed example queue helps maintain
        a full queue when the subsequent computations overall are cheaper than
        parsing.
      parser_num_threads: (Deprecated) The number of threads to parse examples.
      name: Name of resulting op.
  
    Returns:
      A dict of `Tensor` or `SparseTensor` objects for each in `features`.
      If `keep_keys` is `True`, returns tuple of string `Tensor` and above dict.
  
    Raises:
      ValueError: for invalid inputs.
    """
  
    if parser_num_threads:
      # TODO(sibyl-Aix6ihai): Remove on Sept 3 2016.
      logging.warning('parser_num_threads is deprecated, it will be removed on'
                      'Sept 3 2016')
    with ops.op_scope([file_pattern], name, 'read_batch_features') as scope:
      keys, examples = read_keyed_batch_examples(
          file_pattern, batch_size, reader, randomize_input=randomize_input,
          num_epochs=num_epochs, queue_capacity=queue_capacity,
          num_threads=reader_num_threads, read_batch_size=batch_size,
          name=scope)
  
      # Parse the example.
      feature_map = parsing_ops.parse_example(examples, features)
  
      # Lets also add preprocessed tensors into the queue types for each item of
      # the queue.
      tensors_to_enqueue = []
      # Each entry contains the key, and a boolean which indicates whether the
      # tensor was a sparse tensor.
      tensors_mapping = []
      # TODO(sibyl-Aix6ihai): Most of the functionality here is about pushing sparse
      # tensors into a queue. This could be taken care in somewhere else so others
      # can reuse it. Also, QueueBase maybe extended to handle sparse tensors
      # directly.
      for key, tensor in feature_map.iteritems():
        if isinstance(tensor, ops.SparseTensor):
          tensors_mapping.append((key, True))
          tensors_to_enqueue.extend([tensor.indices, tensor.values, tensor.shape])
        else:
          tensors_mapping.append((key, False))
          tensors_to_enqueue.append(tensor)
      tensors_to_enqueue.append(keys)
  
      queue_dtypes = [x.dtype for x in tensors_to_enqueue]
      input_queue = data_flow_ops.FIFOQueue(feature_queue_capacity, queue_dtypes)
  
      # Add a summary op to debug if our feature queue is full or not.
      logging_ops.scalar_summary('queue/parsed_features/%s/fraction_of_%d_full' %
                                 (input_queue.name, feature_queue_capacity),
                                 math_ops.cast(input_queue.size(), dtypes.float32)
                                 * (1. / feature_queue_capacity))
  
      # Add multiple queue runners so that the queue is always full. Adding more
      # than two queue-runners may hog the cpu on the worker to fill up the queue.
      for _ in range(num_queue_runners):
        queue_runner.add_queue_runner(
            queue_runner.QueueRunner(input_queue, [input_queue.enqueue(
                tensors_to_enqueue)]))
  
      dequeued_tensors = input_queue.dequeue()
  
      # Reset shapes on dequeued tensors.
      for i in range(len(tensors_to_enqueue)):
        dequeued_tensors[i].set_shape(tensors_to_enqueue[i].get_shape())
  
      # Recreate feature mapping according to the original dictionary.
      dequeued_feature_map = {}
      index = 0
      for key, is_sparse_tensor in tensors_mapping:
        if is_sparse_tensor:
          # Three tensors are (indices, values, shape).
          dequeued_feature_map[key] = ops.SparseTensor(
              dequeued_tensors[index], dequeued_tensors[index + 1],
              dequeued_tensors[index + 2])
          index += 3
        else:
          dequeued_feature_map[key] = dequeued_tensors[index]
          index += 1
      dequeued_keys = dequeued_tensors[-1]
  
      return dequeued_keys, dequeued_feature_map
function_tokens: def read_keyed_batch_features ( file_pattern , batch_size , features , reader , randomize_input = True , num_epochs = None , queue_capacity = 10000 , reader_num_threads = 1 , feature_queue_capacity = 100 , num_queue_runners = 2 , parser_num_threads = None , name = None ) : if parser_num_threads : # TODO(sibyl-Aix6ihai): Remove on Sept 3 2016. logging . warning ( 'parser_num_threads is deprecated, it will be removed on' 'Sept 3 2016' ) with ops . op_scope ( [ file_pattern ] , name , 'read_batch_features' ) as scope : keys , examples = read_keyed_batch_examples ( file_pattern , batch_size , reader , randomize_input = randomize_input , num_epochs = num_epochs , queue_capacity = queue_capacity , num_threads = reader_num_threads , read_batch_size = batch_size , name = scope ) # Parse the example. feature_map = parsing_ops . parse_example ( examples , features ) # Lets also add preprocessed tensors into the queue types for each item of # the queue. tensors_to_enqueue = [ ] # Each entry contains the key, and a boolean which indicates whether the # tensor was a sparse tensor. tensors_mapping = [ ] # TODO(sibyl-Aix6ihai): Most of the functionality here is about pushing sparse # tensors into a queue. This could be taken care in somewhere else so others # can reuse it. Also, QueueBase maybe extended to handle sparse tensors # directly. for key , tensor in feature_map . iteritems ( ) : if isinstance ( tensor , ops . SparseTensor ) : tensors_mapping . append ( ( key , True ) ) tensors_to_enqueue . extend ( [ tensor . indices , tensor . values , tensor . shape ] ) else : tensors_mapping . append ( ( key , False ) ) tensors_to_enqueue . append ( tensor ) tensors_to_enqueue . append ( keys ) queue_dtypes = [ x . dtype for x in tensors_to_enqueue ] input_queue = data_flow_ops . FIFOQueue ( feature_queue_capacity , queue_dtypes ) # Add a summary op to debug if our feature queue is full or not. logging_ops . scalar_summary ( 'queue/parsed_features/%s/fraction_of_%d_full' % ( input_queue . name , feature_queue_capacity ) , math_ops . cast ( input_queue . size ( ) , dtypes . float32 ) * ( 1. / feature_queue_capacity ) ) # Add multiple queue runners so that the queue is always full. Adding more # than two queue-runners may hog the cpu on the worker to fill up the queue. for _ in range ( num_queue_runners ) : queue_runner . add_queue_runner ( queue_runner . QueueRunner ( input_queue , [ input_queue . enqueue ( tensors_to_enqueue ) ] ) ) dequeued_tensors = input_queue . dequeue ( ) # Reset shapes on dequeued tensors. for i in range ( len ( tensors_to_enqueue ) ) : dequeued_tensors [ i ] . set_shape ( tensors_to_enqueue [ i ] . get_shape ( ) ) # Recreate feature mapping according to the original dictionary. dequeued_feature_map = { } index = 0 for key , is_sparse_tensor in tensors_mapping : if is_sparse_tensor : # Three tensors are (indices, values, shape). dequeued_feature_map [ key ] = ops . SparseTensor ( dequeued_tensors [ index ] , dequeued_tensors [ index + 1 ] , dequeued_tensors [ index + 2 ] ) index += 3 else : dequeued_feature_map [ key ] = dequeued_tensors [ index ] index += 1 dequeued_keys = dequeued_tensors [ - 1 ] return dequeued_keys , dequeued_feature_map