parameters: (self)
docstring: Check memory usage when mulit-processing mode, when 85% prompt warning and 100% raise error.
docstring_summary: Check memory usage when mulit-processing mode, when 85% prompt warning and 100% raise error.
docstring_tokens: Check memory usage when mulit - processing mode when 85% prompt warning and 100% raise error .
function: def __validate_memory_usage(self):
          """
          Check memory usage when mulit-processing mode, when 85% prompt warning and 100% raise error.
          """
          if self.python_multiprocessing:
              # if use num_parallel_workers is to large when python_multiprocessing=True which would cause
              # OOM error get the num_shards
              valid_num_shards = 1
              if isinstance(self.sampler, samplers.DistributedSampler):
                  valid_num_shards = self.sampler.num_shards
              elif self.num_shards is not None:
                  valid_num_shards = self.num_shards
  
              # get process memory usage
              process = psutil.Process(os.getpid())
              process_memory = process.memory_info().rss
              sys_memory_free = psutil.virtual_memory().free
  
              total_memory_maybe_used = process_memory * self.num_parallel_workers * valid_num_shards
              if total_memory_maybe_used / sys_memory_free > 0.85:
                  valid_num_worker = math.floor(sys_memory_free * 0.85 / valid_num_shards / process_memory)
                  valid_num_worker = 1 if valid_num_worker <= 0 else valid_num_worker
                  info = "GeneratorDataset num_parallel_workers: " + str(self.num_parallel_workers) + \
                         " is too large which maybe cause a lot of memory occupation (>85%) or out of memory(OOM) " \
                         "during multi process running. Therefore, it is recommended to reduce num_parallel_workers to " \
                         + str(valid_num_worker) + " or smaller."
                  logger.warning(info)
function_tokens: def __validate_memory_usage ( self ) : if self . python_multiprocessing : # if use num_parallel_workers is to large when python_multiprocessing=True which would cause # OOM error get the num_shards valid_num_shards = 1 if isinstance ( self . sampler , samplers . DistributedSampler ) : valid_num_shards = self . sampler . num_shards elif self . num_shards is not None : valid_num_shards = self . num_shards # get process memory usage process = psutil . Process ( os . getpid ( ) ) process_memory = process . memory_info ( ) . rss sys_memory_free = psutil . virtual_memory ( ) . free total_memory_maybe_used = process_memory * self . num_parallel_workers * valid_num_shards if total_memory_maybe_used / sys_memory_free > 0.85 : valid_num_worker = math . floor ( sys_memory_free * 0.85 / valid_num_shards / process_memory ) valid_num_worker = 1 if valid_num_worker <= 0 else valid_num_worker info = "GeneratorDataset num_parallel_workers: " + str ( self . num_parallel_workers ) + " is too large which maybe cause a lot of memory occupation (>85%) or out of memory(OOM) " "during multi process running. Therefore, it is recommended to reduce num_parallel_workers to " + str ( valid_num_worker ) + " or smaller." logger . warning ( info )