parameters: (cls, mapper_spec)
return_statement: return readers
docstring: Splits query into shards without fetching query results.
  
      Tries as best as it can to split the whole query result set into equal
      shards. Due to difficulty of making the perfect split, resulting shards'
      sizes might differ significantly from each other.
  
      Args:
        mapper_spec: MapperSpec with params containing 'entity_kind'.
          May have 'namespace' in the params as a string containing a single
          namespace. If specified then the input reader will only yield values
          in the given namespace. If 'namespace' is not given then values from
          all namespaces will be yielded. May also have 'batch_size' in the params
          to specify the number of entities to process in each batch.
  
      Returns:
        A list of InputReader objects. If the query results are empty then the
        empty list will be returned. Otherwise, the list will always have a length
        equal to number_of_shards but may be padded with Nones if there are too
        few results for effective sharding.
docstring_summary: Splits query into shards without fetching query results.
docstring_tokens: Splits query into shards without fetching query results .
function: def split_input(cls, mapper_spec):
      """Splits query into shards without fetching query results.
  
      Tries as best as it can to split the whole query result set into equal
      shards. Due to difficulty of making the perfect split, resulting shards'
      sizes might differ significantly from each other.
  
      Args:
        mapper_spec: MapperSpec with params containing 'entity_kind'.
          May have 'namespace' in the params as a string containing a single
          namespace. If specified then the input reader will only yield values
          in the given namespace. If 'namespace' is not given then values from
          all namespaces will be yielded. May also have 'batch_size' in the params
          to specify the number of entities to process in each batch.
  
      Returns:
        A list of InputReader objects. If the query results are empty then the
        empty list will be returned. Otherwise, the list will always have a length
        equal to number_of_shards but may be padded with Nones if there are too
        few results for effective sharding.
      """
      params = _get_params(mapper_spec)
      entity_kind_name = params[cls.ENTITY_KIND_PARAM]
      batch_size = int(params.get(cls.BATCH_SIZE_PARAM, cls._BATCH_SIZE))
      shard_count = mapper_spec.shard_count
      namespace = params.get(cls.NAMESPACE_PARAM)
      app = params.get(cls._APP_PARAM)
      filters = params.get(cls.FILTERS_PARAM)
  
      if namespace is None:
        # It is difficult to efficiently shard large numbers of namespaces because
        # there can be an arbitrary number of them. So the strategy is:
        # 1. if there are a small number of namespaces in the datastore then
        #    generate one KeyRange per namespace per shard and assign each shard a
        #    KeyRange for every namespace. This should lead to nearly perfect
        #    sharding.
        # 2. if there are a large number of namespaces in the datastore then
        #    generate one NamespaceRange per worker. This can lead to very bad
        #    sharding because namespaces can contain very different numbers of
        #    entities and each NamespaceRange may contain very different numbers
        #    of namespaces.
        namespace_query = datastore.Query("__namespace__",
                                          keys_only=True,
                                          _app=app)
        namespace_keys = namespace_query.Get(
            limit=cls.MAX_NAMESPACES_FOR_KEY_SHARD+1)
  
        if len(namespace_keys) > cls.MAX_NAMESPACES_FOR_KEY_SHARD:
          ns_ranges = namespace_range.NamespaceRange.split(n=shard_count,
                                                           contiguous=True,
                                                           _app=app)
          return [cls(entity_kind_name,
                      key_ranges=None,
                      ns_range=ns_range,
                      batch_size=batch_size,
                      filters=filters)
                  for ns_range in ns_ranges]
        elif not namespace_keys:
          return [cls(entity_kind_name,
                      key_ranges=None,
                      ns_range=namespace_range.NamespaceRange(_app=app),
                      batch_size=shard_count,
                      filters=filters)]
        else:
          namespaces = [namespace_key.name() or ""
                        for namespace_key in namespace_keys]
      else:
        namespaces = [namespace]
  
      readers = cls._split_input_from_params(
          app, namespaces, entity_kind_name, params, shard_count)
      if filters:
        for reader in readers:
          reader._filters = filters
      return readers
function_tokens: def split_input ( cls , mapper_spec ) : params = _get_params ( mapper_spec ) entity_kind_name = params [ cls . ENTITY_KIND_PARAM ] batch_size = int ( params . get ( cls . BATCH_SIZE_PARAM , cls . _BATCH_SIZE ) ) shard_count = mapper_spec . shard_count namespace = params . get ( cls . NAMESPACE_PARAM ) app = params . get ( cls . _APP_PARAM ) filters = params . get ( cls . FILTERS_PARAM ) if namespace is None : # It is difficult to efficiently shard large numbers of namespaces because # there can be an arbitrary number of them. So the strategy is: # 1. if there are a small number of namespaces in the datastore then #    generate one KeyRange per namespace per shard and assign each shard a #    KeyRange for every namespace. This should lead to nearly perfect #    sharding. # 2. if there are a large number of namespaces in the datastore then #    generate one NamespaceRange per worker. This can lead to very bad #    sharding because namespaces can contain very different numbers of #    entities and each NamespaceRange may contain very different numbers #    of namespaces. namespace_query = datastore . Query ( "__namespace__" , keys_only = True , _app = app ) namespace_keys = namespace_query . Get ( limit = cls . MAX_NAMESPACES_FOR_KEY_SHARD + 1 ) if len ( namespace_keys ) > cls . MAX_NAMESPACES_FOR_KEY_SHARD : ns_ranges = namespace_range . NamespaceRange . split ( n = shard_count , contiguous = True , _app = app ) return [ cls ( entity_kind_name , key_ranges = None , ns_range = ns_range , batch_size = batch_size , filters = filters ) for ns_range in ns_ranges ] elif not namespace_keys : return [ cls ( entity_kind_name , key_ranges = None , ns_range = namespace_range . NamespaceRange ( _app = app ) , batch_size = shard_count , filters = filters ) ] else : namespaces = [ namespace_key . name ( ) or "" for namespace_key in namespace_keys ] else : namespaces = [ namespace ] readers = cls . _split_input_from_params ( app , namespaces , entity_kind_name , params , shard_count ) if filters : for reader in readers : reader . _filters = filters return readers