parameters: (self, dataset, preprocessing_transform=None,
                   cache_dir=None, num_workers=None, transform=None,
                   no_progress: bool = False)
docstring: A wrapper dataset that applies a preprocessing transform to a given base
          dataset. The result of the preprocessing transform will be cached to
          disk.
  
          The base dataset should support a `get_data(idx)` method that returns
          the data to be preprocessed. If such method is not found, then its
          `__getitem__(idx)` method will be used instead.
  
          The base dataset can optionally support a `get_attributes(idx)` method
          that returns the data that should not be preprocessed. If such method is
          not found, an empty dict will be used as attributes. The `__getitem__`
          of `ProcessedDataset` will contain both the data and the attributes.
  
          The base dataset can optionally support a `get_cache_key(idx)` method
          that returns the string key to use for caching. If such method is not
          found, the index (as a string) will be used as cache key.
  
          Note: if CUDA is used in preprocessing, `num_workers` must be set to 0.
  
          Args:
              dataset (torch.utils.data.Dataset):
                  The base dataset to preprocess.
              cache_dir (str):
                  Path to the cached preprocessed data. Must be given if
                  `preprocessing_transform` is not None.
              num_workers (int):
                  Number of process used in parallel for preprocessing (default:
                  number of cores)
              preprocessing_transform (Callable):
                  Called on the outputs of get_data over the indices from 0 to
                  `len(self)` during the construction of the dataset, the
                  preprocessed outputs are then cached to 'cache_dir'.
              transform (Callable):
                  Called on the preprocessed data at `__getitem__`.
                  The result of this function is not cached, unlike
                  `preprocessing_transform`.
              no_progress (bool): Disable tqdm progress bar for preprocessing.
docstring_summary: A wrapper dataset that applies a preprocessing transform to a given base
          dataset. The result of the preprocessing transform will be cached to
          disk.
docstring_tokens: A wrapper dataset that applies a preprocessing transform to a given base dataset . The result of the preprocessing transform will be cached to disk .
function: def __init__(self, dataset, preprocessing_transform=None,
                   cache_dir=None, num_workers=None, transform=None,
                   no_progress: bool = False):
          """
          A wrapper dataset that applies a preprocessing transform to a given base
          dataset. The result of the preprocessing transform will be cached to
          disk.
  
          The base dataset should support a `get_data(idx)` method that returns
          the data to be preprocessed. If such method is not found, then its
          `__getitem__(idx)` method will be used instead.
  
          The base dataset can optionally support a `get_attributes(idx)` method
          that returns the data that should not be preprocessed. If such method is
          not found, an empty dict will be used as attributes. The `__getitem__`
          of `ProcessedDataset` will contain both the data and the attributes.
  
          The base dataset can optionally support a `get_cache_key(idx)` method
          that returns the string key to use for caching. If such method is not
          found, the index (as a string) will be used as cache key.
  
          Note: if CUDA is used in preprocessing, `num_workers` must be set to 0.
  
          Args:
              dataset (torch.utils.data.Dataset):
                  The base dataset to preprocess.
              cache_dir (str):
                  Path to the cached preprocessed data. Must be given if
                  `preprocessing_transform` is not None.
              num_workers (int):
                  Number of process used in parallel for preprocessing (default:
                  number of cores)
              preprocessing_transform (Callable):
                  Called on the outputs of get_data over the indices from 0 to
                  `len(self)` during the construction of the dataset, the
                  preprocessed outputs are then cached to 'cache_dir'.
              transform (Callable):
                  Called on the preprocessed data at `__getitem__`.
                  The result of this function is not cached, unlike
                  `preprocessing_transform`.
              no_progress (bool): Disable tqdm progress bar for preprocessing.
          """
          # TODO: Consider integrating combination into `ProcessedDataset`.
  
          self.dataset = dataset
          self.transform = transform
  
          if preprocessing_transform is not None:
              desc = 'Applying preprocessing'
  
              assert cache_dir is not None, 'Cache directory is not given'
  
              self.cache_convert = Cache(
                  preprocessing_transform,
                  cache_dir=cache_dir,
                  cache_key=_get_hash(repr(preprocessing_transform))
              )
  
              uncached = [idx for idx in range(len(self)) if
                          self.get_cache_key(idx) not in
                          self.cache_convert.cached_ids]
              if len(uncached) > 0:
                  if num_workers == 0:
                      with torch.no_grad():
                          for idx in tqdm(range(len(self)), desc=desc,
                                          disable=no_progress):
                              key = self.get_cache_key(idx)
                              data = self._get_base_data(idx)
                              self.cache_convert(key, data)
                  else:
                      p = Pool(num_workers)
                      iterator = p.imap_unordered(
                          _preprocess_task,
                          [(idx, self._get_base_data, self.get_cache_key,
                            self.cache_convert)
                           for idx in uncached])
                      for i in tqdm(range(len(uncached)), desc=desc,
                                    disable=no_progress):
                          next(iterator)
          else:
              self.cache_convert = None
function_tokens: def __init__ ( self , dataset , preprocessing_transform = None , cache_dir = None , num_workers = None , transform = None , no_progress : bool = False ) : # TODO: Consider integrating combination into `ProcessedDataset`. self . dataset = dataset self . transform = transform if preprocessing_transform is not None : desc = 'Applying preprocessing' assert cache_dir is not None , 'Cache directory is not given' self . cache_convert = Cache ( preprocessing_transform , cache_dir = cache_dir , cache_key = _get_hash ( repr ( preprocessing_transform ) ) ) uncached = [ idx for idx in range ( len ( self ) ) if self . get_cache_key ( idx ) not in self . cache_convert . cached_ids ] if len ( uncached ) > 0 : if num_workers == 0 : with torch . no_grad ( ) : for idx in tqdm ( range ( len ( self ) ) , desc = desc , disable = no_progress ) : key = self . get_cache_key ( idx ) data = self . _get_base_data ( idx ) self . cache_convert ( key , data ) else : p = Pool ( num_workers ) iterator = p . imap_unordered ( _preprocess_task , [ ( idx , self . _get_base_data , self . get_cache_key , self . cache_convert ) for idx in uncached ] ) for i in tqdm ( range ( len ( uncached ) ) , desc = desc , disable = no_progress ) : next ( iterator ) else : self . cache_convert = None