parameters: (self)
docstring: Loop to selectively push PGs below their min_size and test that recovery
          still occurs.
docstring_summary: Loop to selectively push PGs below their min_size and test that recovery
          still occurs.
docstring_tokens: Loop to selectively push PGs below their min_size and test that recovery still occurs .
function: def test_pool_min_size(self):
          """
          Loop to selectively push PGs below their min_size and test that recovery
          still occurs.
          """
          self.log("test_pool_min_size")
          self.all_up()
          self.ceph_manager.wait_for_recovery(
              timeout=self.config.get('timeout')
              )
  
          minout = int(self.config.get("min_out", 1))
          minlive = int(self.config.get("min_live", 2))
          mindead = int(self.config.get("min_dead", 1))
          self.log("doing min_size thrashing")
          self.ceph_manager.wait_for_clean(timeout=60)
          assert self.ceph_manager.is_clean(), \
              'not clean before minsize thrashing starts'
          while not self.stopping:
              # look up k and m from all the pools on each loop, in case it
              # changes as the cluster runs
              k = 0
              m = 99
              has_pools = False
              pools_json = self.ceph_manager.get_osd_dump_json()['pools']
  
              for pool_json in pools_json:
                  pool = pool_json['pool_name']
                  has_pools = True
                  pool_type = pool_json['type']  # 1 for rep, 3 for ec
                  min_size = pool_json['min_size']
                  self.log("pool {pool} min_size is {min_size}".format(pool=pool,min_size=min_size))
                  try:
                      ec_profile = self.ceph_manager.get_pool_property(pool, 'erasure_code_profile')
                      if pool_type != PoolType.ERASURE_CODED:
                          continue
                      ec_profile = pool_json['erasure_code_profile']
                      ec_profile_json = self.ceph_manager.raw_cluster_cmd(
                          'osd',
                          'erasure-code-profile',
                          'get',
                          ec_profile,
                          '--format=json')
                      ec_json = json.loads(ec_profile_json)
                      local_k = int(ec_json['k'])
                      local_m = int(ec_json['m'])
                      self.log("pool {pool} local_k={k} local_m={m}".format(pool=pool,
                                                                            k=local_k, m=local_m))
                      if local_k > k:
                          self.log("setting k={local_k} from previous {k}".format(local_k=local_k, k=k))
                          k = local_k
                      if local_m < m:
                          self.log("setting m={local_m} from previous {m}".format(local_m=local_m, m=m))
                          m = local_m
                  except CommandFailedError:
                      self.log("failed to read erasure_code_profile. %s was likely removed", pool)
                      continue
  
              if has_pools :
                  self.log("using k={k}, m={m}".format(k=k,m=m))
              else:
                  self.log("No pools yet, waiting")
                  time.sleep(5)
                  continue
                  
              if minout > len(self.out_osds): # kill OSDs and mark out
                  self.log("forced to out an osd")
                  self.kill_osd(mark_out=True)
                  continue
              elif mindead > len(self.dead_osds): # kill OSDs but force timeout
                  self.log("forced to kill an osd")
                  self.kill_osd()
                  continue
              else: # make mostly-random choice to kill or revive OSDs
                  minup = max(minlive, k)
                  rand_val = random.uniform(0, 1)
                  self.log("choosing based on number of live OSDs and rand val {rand}".\
                           format(rand=rand_val))
                  if len(self.live_osds) > minup+1 and rand_val < 0.5:
                      # chose to knock out as many OSDs as we can w/out downing PGs
                      
                      most_killable = min(len(self.live_osds) - minup, m)
                      self.log("chose to kill {n} OSDs".format(n=most_killable))
                      for i in range(1, most_killable):
                          self.kill_osd(mark_out=True)
                      time.sleep(10)
                      # try a few times since there might be a concurrent pool
                      # creation or deletion
                      with safe_while(
                              sleep=5, tries=5,
                              action='check for active or peered') as proceed:
                          while proceed():
                              if self.ceph_manager.all_active_or_peered():
                                  break
                              self.log('not all PGs are active or peered')
                  else: # chose to revive OSDs, bring up a random fraction of the dead ones
                      self.log("chose to revive osds")
                      for i in range(1, int(rand_val * len(self.dead_osds))):
                          self.revive_osd(i)
  
              # let PGs repair themselves or our next knockout might kill one
              self.ceph_manager.wait_for_clean(timeout=self.config.get('timeout'))
   
          # / while not self.stopping
          self.all_up_in()
   
          self.ceph_manager.wait_for_recovery(
              timeout=self.config.get('timeout')
              )
function_tokens: def test_pool_min_size ( self ) : self . log ( "test_pool_min_size" ) self . all_up ( ) self . ceph_manager . wait_for_recovery ( timeout = self . config . get ( 'timeout' ) ) minout = int ( self . config . get ( "min_out" , 1 ) ) minlive = int ( self . config . get ( "min_live" , 2 ) ) mindead = int ( self . config . get ( "min_dead" , 1 ) ) self . log ( "doing min_size thrashing" ) self . ceph_manager . wait_for_clean ( timeout = 60 ) assert self . ceph_manager . is_clean ( ) , 'not clean before minsize thrashing starts' while not self . stopping : # look up k and m from all the pools on each loop, in case it # changes as the cluster runs k = 0 m = 99 has_pools = False pools_json = self . ceph_manager . get_osd_dump_json ( ) [ 'pools' ] for pool_json in pools_json : pool = pool_json [ 'pool_name' ] has_pools = True pool_type = pool_json [ 'type' ] # 1 for rep, 3 for ec min_size = pool_json [ 'min_size' ] self . log ( "pool {pool} min_size is {min_size}" . format ( pool = pool , min_size = min_size ) ) try : ec_profile = self . ceph_manager . get_pool_property ( pool , 'erasure_code_profile' ) if pool_type != PoolType . ERASURE_CODED : continue ec_profile = pool_json [ 'erasure_code_profile' ] ec_profile_json = self . ceph_manager . raw_cluster_cmd ( 'osd' , 'erasure-code-profile' , 'get' , ec_profile , '--format=json' ) ec_json = json . loads ( ec_profile_json ) local_k = int ( ec_json [ 'k' ] ) local_m = int ( ec_json [ 'm' ] ) self . log ( "pool {pool} local_k={k} local_m={m}" . format ( pool = pool , k = local_k , m = local_m ) ) if local_k > k : self . log ( "setting k={local_k} from previous {k}" . format ( local_k = local_k , k = k ) ) k = local_k if local_m < m : self . log ( "setting m={local_m} from previous {m}" . format ( local_m = local_m , m = m ) ) m = local_m except CommandFailedError : self . log ( "failed to read erasure_code_profile. %s was likely removed" , pool ) continue if has_pools : self . log ( "using k={k}, m={m}" . format ( k = k , m = m ) ) else : self . log ( "No pools yet, waiting" ) time . sleep ( 5 ) continue if minout > len ( self . out_osds ) : # kill OSDs and mark out self . log ( "forced to out an osd" ) self . kill_osd ( mark_out = True ) continue elif mindead > len ( self . dead_osds ) : # kill OSDs but force timeout self . log ( "forced to kill an osd" ) self . kill_osd ( ) continue else : # make mostly-random choice to kill or revive OSDs minup = max ( minlive , k ) rand_val = random . uniform ( 0 , 1 ) self . log ( "choosing based on number of live OSDs and rand val {rand}" . format ( rand = rand_val ) ) if len ( self . live_osds ) > minup + 1 and rand_val < 0.5 : # chose to knock out as many OSDs as we can w/out downing PGs most_killable = min ( len ( self . live_osds ) - minup , m ) self . log ( "chose to kill {n} OSDs" . format ( n = most_killable ) ) for i in range ( 1 , most_killable ) : self . kill_osd ( mark_out = True ) time . sleep ( 10 ) # try a few times since there might be a concurrent pool # creation or deletion with safe_while ( sleep = 5 , tries = 5 , action = 'check for active or peered' ) as proceed : while proceed ( ) : if self . ceph_manager . all_active_or_peered ( ) : break self . log ( 'not all PGs are active or peered' ) else : # chose to revive OSDs, bring up a random fraction of the dead ones self . log ( "chose to revive osds" ) for i in range ( 1 , int ( rand_val * len ( self . dead_osds ) ) ) : self . revive_osd ( i ) # let PGs repair themselves or our next knockout might kill one self . ceph_manager . wait_for_clean ( timeout = self . config . get ( 'timeout' ) ) # / while not self.stopping self . all_up_in ( ) self . ceph_manager . wait_for_recovery ( timeout = self . config . get ( 'timeout' ) )