parameters: (ctx, config)
docstring: Handle the creation and removal of a ceph cluster.
  
      On startup:
          Create directories needed for the cluster.
          Create remote journals for all osds.
          Create and set keyring.
          Copy the monmap to the test systems.
          Setup mon nodes.
          Setup mds nodes.
          Mkfs osd nodes.
          Add keyring information to monmaps
          Mkfs mon nodes.
  
      On exit:
          If errors occurred, extract a failure message and store in ctx.summary.
          Unmount all test files and temporary journaling files.
          Save the monitor information and archive all ceph logs.
          Cleanup the keyring setup, and remove all monitor map and data files left over.
  
      :param ctx: Context
      :param config: Configuration
docstring_summary: Handle the creation and removal of a ceph cluster.
docstring_tokens: Handle the creation and removal of a ceph cluster .
function: def cluster(ctx, config):
      """
      Handle the creation and removal of a ceph cluster.
  
      On startup:
          Create directories needed for the cluster.
          Create remote journals for all osds.
          Create and set keyring.
          Copy the monmap to the test systems.
          Setup mon nodes.
          Setup mds nodes.
          Mkfs osd nodes.
          Add keyring information to monmaps
          Mkfs mon nodes.
  
      On exit:
          If errors occurred, extract a failure message and store in ctx.summary.
          Unmount all test files and temporary journaling files.
          Save the monitor information and archive all ceph logs.
          Cleanup the keyring setup, and remove all monitor map and data files left over.
  
      :param ctx: Context
      :param config: Configuration
      """
      if ctx.config.get('use_existing_cluster', False) is True:
          log.info("'use_existing_cluster' is true; skipping cluster creation")
          yield
  
      testdir = teuthology.get_testdir(ctx)
      cluster_name = config['cluster']
      data_dir = '{tdir}/{cluster}.data'.format(tdir=testdir, cluster=cluster_name)
      log.info('Creating ceph cluster %s...', cluster_name)
      log.info('config %s', config)
      log.info('ctx.config %s', ctx.config)
      run.wait(
          ctx.cluster.run(
              args=[
                  'install', '-d', '-m0755', '--',
                  data_dir,
              ],
              wait=False,
          )
      )
  
      run.wait(
          ctx.cluster.run(
              args=[
                  'sudo',
                  'install', '-d', '-m0777', '--', '/var/run/ceph',
              ],
              wait=False,
          )
      )
  
      devs_to_clean = {}
      remote_to_roles_to_devs = {}
      osds = ctx.cluster.only(teuthology.is_type('osd', cluster_name))
      for remote, roles_for_host in osds.remotes.items():
          devs = teuthology.get_scratch_devices(remote)
          roles_to_devs = assign_devs(
              teuthology.cluster_roles_of_type(roles_for_host, 'osd', cluster_name), devs
          )
          devs_to_clean[remote] = []
          log.info('osd dev map: {}'.format(roles_to_devs))
          assert roles_to_devs, \
              "remote {} has osd roles, but no osd devices were specified!".format(remote.hostname)
          remote_to_roles_to_devs[remote] = roles_to_devs
      log.info("remote_to_roles_to_devs: {}".format(remote_to_roles_to_devs))
      for osd_role, dev_name in remote_to_roles_to_devs.items():
          assert dev_name, "{} has no associated device!".format(osd_role)
  
      log.info('Generating config...')
      remotes_and_roles = ctx.cluster.remotes.items()
      roles = [role_list for (remote, role_list) in remotes_and_roles]
      ips = [host for (host, port) in
             (remote.ssh.get_transport().getpeername() for (remote, role_list) in remotes_and_roles)]
      mons = get_mons(
          roles, ips, cluster_name,
          mon_bind_msgr2=config.get('mon_bind_msgr2'),
          mon_bind_addrvec=config.get('mon_bind_addrvec'),
          )
      conf = skeleton_config(
          ctx, roles=roles, ips=ips, mons=mons, cluster=cluster_name,
      )
      for section, keys in config['conf'].items():
          for key, value in keys.items():
              log.info("[%s] %s = %s" % (section, key, value))
              if section not in conf:
                  conf[section] = {}
              conf[section][key] = value
  
      if not hasattr(ctx, 'ceph'):
          ctx.ceph = {}
      ctx.ceph[cluster_name] = argparse.Namespace()
      ctx.ceph[cluster_name].conf = conf
      ctx.ceph[cluster_name].mons = mons
  
      default_keyring = '/etc/ceph/{cluster}.keyring'.format(cluster=cluster_name)
      keyring_path = config.get('keyring_path', default_keyring)
  
      coverage_dir = '{tdir}/archive/coverage'.format(tdir=testdir)
  
      firstmon = teuthology.get_first_mon(ctx, config, cluster_name)
  
      log.info('Setting up %s...' % firstmon)
      ctx.cluster.only(firstmon).run(
          args=[
              'sudo',
              'adjust-ulimits',
              'ceph-coverage',
              coverage_dir,
              'ceph-authtool',
              '--create-keyring',
              keyring_path,
          ],
      )
      ctx.cluster.only(firstmon).run(
          args=[
              'sudo',
              'adjust-ulimits',
              'ceph-coverage',
              coverage_dir,
              'ceph-authtool',
              '--gen-key',
              '--name=mon.',
              keyring_path,
          ],
      )
      ctx.cluster.only(firstmon).run(
          args=[
              'sudo',
              'chmod',
              '0644',
              keyring_path,
          ],
      )
      (mon0_remote,) = ctx.cluster.only(firstmon).remotes.keys()
      monmap_path = '{tdir}/{cluster}.monmap'.format(tdir=testdir,
                                                     cluster=cluster_name)
      fsid = create_simple_monmap(
          ctx,
          remote=mon0_remote,
          conf=conf,
          mons=mons,
          path=monmap_path,
          mon_bind_addrvec=config.get('mon_bind_addrvec'),
      )
      ctx.ceph[cluster_name].fsid = fsid
      if not 'global' in conf:
          conf['global'] = {}
      conf['global']['fsid'] = fsid
  
      default_conf_path = '/etc/ceph/{cluster}.conf'.format(cluster=cluster_name)
      conf_path = config.get('conf_path', default_conf_path)
      log.info('Writing %s for FSID %s...' % (conf_path, fsid))
      write_conf(ctx, conf_path, cluster_name)
  
      log.info('Creating admin key on %s...' % firstmon)
      ctx.cluster.only(firstmon).run(
          args=[
              'sudo',
              'adjust-ulimits',
              'ceph-coverage',
              coverage_dir,
              'ceph-authtool',
              '--gen-key',
              '--name=client.admin',
              '--cap', 'mon', 'allow *',
              '--cap', 'osd', 'allow *',
              '--cap', 'mds', 'allow *',
              '--cap', 'mgr', 'allow *',
              keyring_path,
          ],
      )
  
      log.info('Copying monmap to all nodes...')
      keyring = mon0_remote.read_file(keyring_path)
      monmap = mon0_remote.read_file(monmap_path)
  
      for rem in ctx.cluster.remotes.keys():
          # copy mon key and initial monmap
          log.info('Sending monmap to node {remote}'.format(remote=rem))
          rem.write_file(keyring_path, keyring, mode='0644', sudo=True)
          rem.write_file(monmap_path, monmap)
  
      log.info('Setting up mon nodes...')
      mons = ctx.cluster.only(teuthology.is_type('mon', cluster_name))
  
      if not config.get('skip_mgr_daemons', False):
          log.info('Setting up mgr nodes...')
          mgrs = ctx.cluster.only(teuthology.is_type('mgr', cluster_name))
          for remote, roles_for_host in mgrs.remotes.items():
              for role in teuthology.cluster_roles_of_type(roles_for_host, 'mgr',
                                                           cluster_name):
                  _, _, id_ = teuthology.split_role(role)
                  mgr_dir = DATA_PATH.format(
                      type_='mgr', cluster=cluster_name, id_=id_)
                  remote.run(
                      args=[
                          'sudo',
                          'mkdir',
                          '-p',
                          mgr_dir,
                          run.Raw('&&'),
                          'sudo',
                          'adjust-ulimits',
                          'ceph-coverage',
                          coverage_dir,
                          'ceph-authtool',
                          '--create-keyring',
                          '--gen-key',
                          '--name=mgr.{id}'.format(id=id_),
                          mgr_dir + '/keyring',
                      ],
                  )
  
      log.info('Setting up mds nodes...')
      mdss = ctx.cluster.only(teuthology.is_type('mds', cluster_name))
      for remote, roles_for_host in mdss.remotes.items():
          for role in teuthology.cluster_roles_of_type(roles_for_host, 'mds',
                                                       cluster_name):
              _, _, id_ = teuthology.split_role(role)
              mds_dir = DATA_PATH.format(
                  type_='mds', cluster=cluster_name, id_=id_)
              remote.run(
                  args=[
                      'sudo',
                      'mkdir',
                      '-p',
                      mds_dir,
                      run.Raw('&&'),
                      'sudo',
                      'adjust-ulimits',
                      'ceph-coverage',
                      coverage_dir,
                      'ceph-authtool',
                      '--create-keyring',
                      '--gen-key',
                      '--name=mds.{id}'.format(id=id_),
                      mds_dir + '/keyring',
                  ],
              )
              remote.run(args=[
                  'sudo', 'chown', '-R', 'ceph:ceph', mds_dir
              ])
  
      cclient.create_keyring(ctx, cluster_name)
      log.info('Running mkfs on osd nodes...')
  
      if not hasattr(ctx, 'disk_config'):
          ctx.disk_config = argparse.Namespace()
      if not hasattr(ctx.disk_config, 'remote_to_roles_to_dev'):
          ctx.disk_config.remote_to_roles_to_dev = {}
      if not hasattr(ctx.disk_config, 'remote_to_roles_to_dev_mount_options'):
          ctx.disk_config.remote_to_roles_to_dev_mount_options = {}
      if not hasattr(ctx.disk_config, 'remote_to_roles_to_dev_fstype'):
          ctx.disk_config.remote_to_roles_to_dev_fstype = {}
  
      teuthology.deep_merge(ctx.disk_config.remote_to_roles_to_dev, remote_to_roles_to_devs)
  
      log.info("ctx.disk_config.remote_to_roles_to_dev: {r}".format(r=str(ctx.disk_config.remote_to_roles_to_dev)))
      for remote, roles_for_host in osds.remotes.items():
          roles_to_devs = remote_to_roles_to_devs[remote]
  
          for role in teuthology.cluster_roles_of_type(roles_for_host, 'osd', cluster_name):
              _, _, id_ = teuthology.split_role(role)
              mnt_point = DATA_PATH.format(
                  type_='osd', cluster=cluster_name, id_=id_)
              remote.run(
                  args=[
                      'sudo',
                      'mkdir',
                      '-p',
                      mnt_point,
                  ])
              log.info('roles_to_devs: {}'.format(roles_to_devs))
              log.info('role: {}'.format(role))
              if roles_to_devs.get(role):
                  dev = roles_to_devs[role]
                  fs = config.get('fs')
                  package = None
                  mkfs_options = config.get('mkfs_options')
                  mount_options = config.get('mount_options')
                  if fs == 'btrfs':
                      # package = 'btrfs-tools'
                      if mount_options is None:
                          mount_options = ['noatime', 'user_subvol_rm_allowed']
                      if mkfs_options is None:
                          mkfs_options = ['-m', 'single',
                                          '-l', '32768',
                                          '-n', '32768']
                  if fs == 'xfs':
                      # package = 'xfsprogs'
                      if mount_options is None:
                          mount_options = ['noatime']
                      if mkfs_options is None:
                          mkfs_options = ['-f', '-i', 'size=2048']
                  if fs == 'ext4' or fs == 'ext3':
                      if mount_options is None:
                          mount_options = ['noatime', 'user_xattr']
  
                  if mount_options is None:
                      mount_options = []
                  if mkfs_options is None:
                      mkfs_options = []
                  mkfs = ['mkfs.%s' % fs] + mkfs_options
                  log.info('%s on %s on %s' % (mkfs, dev, remote))
                  if package is not None:
                      remote.sh('sudo apt-get install -y %s' % package)
  
                  try:
                      remote.run(args=['yes', run.Raw('|')] + ['sudo'] + mkfs + [dev])
                  except run.CommandFailedError:
                      # Newer btfs-tools doesn't prompt for overwrite, use -f
                      if '-f' not in mount_options:
                          mkfs_options.append('-f')
                          mkfs = ['mkfs.%s' % fs] + mkfs_options
                          log.info('%s on %s on %s' % (mkfs, dev, remote))
                      remote.run(args=['yes', run.Raw('|')] + ['sudo'] + mkfs + [dev])
  
                  log.info('mount %s on %s -o %s' % (dev, remote,
                                                     ','.join(mount_options)))
                  remote.run(
                      args=[
                          'sudo',
                          'mount',
                          '-t', fs,
                          '-o', ','.join(mount_options),
                          dev,
                          mnt_point,
                      ]
                  )
                  remote.run(
                      args=[
                          'sudo', '/sbin/restorecon', mnt_point,
                      ],
                      check_status=False,
                  )
                  if not remote in ctx.disk_config.remote_to_roles_to_dev_mount_options:
                      ctx.disk_config.remote_to_roles_to_dev_mount_options[remote] = {}
                  ctx.disk_config.remote_to_roles_to_dev_mount_options[remote][role] = mount_options
                  if not remote in ctx.disk_config.remote_to_roles_to_dev_fstype:
                      ctx.disk_config.remote_to_roles_to_dev_fstype[remote] = {}
                  ctx.disk_config.remote_to_roles_to_dev_fstype[remote][role] = fs
                  devs_to_clean[remote].append(mnt_point)
  
          for role in teuthology.cluster_roles_of_type(roles_for_host, 'osd', cluster_name):
              _, _, id_ = teuthology.split_role(role)
              try:
                  args = ['sudo',
                          'MALLOC_CHECK_=3',
                          'adjust-ulimits',
                          'ceph-coverage', coverage_dir,
                          'ceph-osd',
                          '--no-mon-config',
                          '--cluster', cluster_name,
                          '--mkfs',
                          '--mkkey',
                          '-i', id_,
                          '--monmap', monmap_path]
                  log_path = f'/var/log/ceph/{cluster_name}-osd.{id_}.log'
                  create_log_cmd, args = \
                      maybe_redirect_stderr(config, 'osd', args, log_path)
                  if create_log_cmd:
                      remote.sh(create_log_cmd)
                  remote.run(args=args)
              except run.CommandFailedError:
                  # try without --no-mon-config.. this may be an upgrade test
                  remote.run(
                      args=[
                          'sudo',
                          'MALLOC_CHECK_=3',
                          'adjust-ulimits',
                          'ceph-coverage',
                          coverage_dir,
                          'ceph-osd',
                          '--cluster',
                          cluster_name,
                          '--mkfs',
                          '--mkkey',
                          '-i', id_,
                      '--monmap', monmap_path,
                      ],
                  )
              mnt_point = DATA_PATH.format(
                  type_='osd', cluster=cluster_name, id_=id_)
              remote.run(args=[
                  'sudo', 'chown', '-R', 'ceph:ceph', mnt_point
              ])
  
      log.info('Reading keys from all nodes...')
      keys_fp = BytesIO()
      keys = []
      for remote, roles_for_host in ctx.cluster.remotes.items():
          for type_ in ['mgr',  'mds', 'osd']:
              if type_ == 'mgr' and config.get('skip_mgr_daemons', False):
                  continue
              for role in teuthology.cluster_roles_of_type(roles_for_host, type_, cluster_name):
                  _, _, id_ = teuthology.split_role(role)
                  data = remote.read_file(
                      os.path.join(
                          DATA_PATH.format(
                              type_=type_, id_=id_, cluster=cluster_name),
                          'keyring',
                      ),
                      sudo=True,
                  )
                  keys.append((type_, id_, data))
                  keys_fp.write(data)
      for remote, roles_for_host in ctx.cluster.remotes.items():
          for role in teuthology.cluster_roles_of_type(roles_for_host, 'client', cluster_name):
              _, _, id_ = teuthology.split_role(role)
              data = remote.read_file(
                  '/etc/ceph/{cluster}.client.{id}.keyring'.format(id=id_, cluster=cluster_name)
              )
              keys.append(('client', id_, data))
              keys_fp.write(data)
  
      log.info('Adding keys to all mons...')
      writes = mons.run(
          args=[
              'sudo', 'tee', '-a',
              keyring_path,
          ],
          stdin=run.PIPE,
          wait=False,
          stdout=BytesIO(),
      )
      keys_fp.seek(0)
      teuthology.feed_many_stdins_and_close(keys_fp, writes)
      run.wait(writes)
      for type_, id_, data in keys:
          run.wait(
              mons.run(
                  args=[
                           'sudo',
                           'adjust-ulimits',
                           'ceph-coverage',
                           coverage_dir,
                           'ceph-authtool',
                           keyring_path,
                           '--name={type}.{id}'.format(
                               type=type_,
                               id=id_,
                           ),
                       ] + list(generate_caps(type_)),
                  wait=False,
              ),
          )
  
      log.info('Running mkfs on mon nodes...')
      for remote, roles_for_host in mons.remotes.items():
          for role in teuthology.cluster_roles_of_type(roles_for_host, 'mon', cluster_name):
              _, _, id_ = teuthology.split_role(role)
              mnt_point = DATA_PATH.format(
                  type_='mon', id_=id_, cluster=cluster_name)
              remote.run(
                  args=[
                      'sudo',
                      'mkdir',
                      '-p',
                      mnt_point,
                  ],
              )
              remote.run(
                  args=[
                      'sudo',
                      'adjust-ulimits',
                      'ceph-coverage',
                      coverage_dir,
                      'ceph-mon',
                      '--cluster', cluster_name,
                      '--mkfs',
                      '-i', id_,
                      '--monmap', monmap_path,
                      '--keyring', keyring_path,
                  ],
              )
              remote.run(args=[
                  'sudo', 'chown', '-R', 'ceph:ceph', mnt_point
              ])
  
      run.wait(
          mons.run(
              args=[
                  'rm',
                  '--',
                  monmap_path,
              ],
              wait=False,
          ),
      )
  
      try:
          yield
      except Exception:
          # we need to know this below
          ctx.summary['success'] = False
          raise
      finally:
          (mon0_remote,) = ctx.cluster.only(firstmon).remotes.keys()
  
          log.info('Checking cluster log for badness...')
  
          def first_in_ceph_log(pattern, excludes):
              """
              Find the first occurrence of the pattern specified in the Ceph log,
              Returns None if none found.
  
              :param pattern: Pattern scanned for.
              :param excludes: Patterns to ignore.
              :return: First line of text (or None if not found)
              """
              args = [
                  'sudo',
                  'egrep', pattern,
                  '/var/log/ceph/{cluster}.log'.format(cluster=cluster_name),
              ]
              for exclude in excludes:
                  args.extend([run.Raw('|'), 'egrep', '-v', exclude])
              args.extend([
                  run.Raw('|'), 'head', '-n', '1',
              ])
              stdout = mon0_remote.sh(args)
              return stdout or None
  
          if first_in_ceph_log('\[ERR\]|\[WRN\]|\[SEC\]',
                               config['log_ignorelist']) is not None:
              log.warning('Found errors (ERR|WRN|SEC) in cluster log')
              ctx.summary['success'] = False
              # use the most severe problem as the failure reason
              if 'failure_reason' not in ctx.summary:
                  for pattern in ['\[SEC\]', '\[ERR\]', '\[WRN\]']:
                      match = first_in_ceph_log(pattern, config['log_ignorelist'])
                      if match is not None:
                          ctx.summary['failure_reason'] = \
                              '"{match}" in cluster log'.format(
                                  match=match.rstrip('\n'),
                              )
                          break
  
          for remote, dirs in devs_to_clean.items():
              for dir_ in dirs:
                  log.info('Unmounting %s on %s' % (dir_, remote))
                  try:
                      remote.run(
                          args=[
                              'sync',
                              run.Raw('&&'),
                              'sudo',
                              'umount',
                              '-f',
                              dir_
                          ]
                      )
                  except Exception as e:
                      remote.run(args=[
                          'sudo',
                          run.Raw('PATH=/usr/sbin:$PATH'),
                          'lsof',
                          run.Raw(';'),
                          'ps', 'auxf',
                      ])
                      raise e
  
          if ctx.archive is not None and \
                  not (ctx.config.get('archive-on-error') and ctx.summary['success']):
  
              # archive mon data, too
              log.info('Archiving mon data...')
              path = os.path.join(ctx.archive, 'data')
              try:
                  os.makedirs(path)
              except OSError as e:
                  if e.errno == errno.EEXIST:
                      pass
                  else:
                      raise
              for remote, roles in mons.remotes.items():
                  for role in roles:
                      is_mon = teuthology.is_type('mon', cluster_name)
                      if is_mon(role):
                          _, _, id_ = teuthology.split_role(role)
                          mon_dir = DATA_PATH.format(
                              type_='mon', id_=id_, cluster=cluster_name)
                          teuthology.pull_directory_tarball(
                              remote,
                              mon_dir,
                              path + '/' + role + '.tgz')
  
          log.info('Cleaning ceph cluster...')
          run.wait(
              ctx.cluster.run(
                  args=[
                      'sudo',
                      'rm',
                      '-rf',
                      '--',
                      conf_path,
                      keyring_path,
                      data_dir,
                      monmap_path,
                      run.Raw('{tdir}/../*.pid'.format(tdir=testdir)),
                  ],
                  wait=False,
              ),
          )
function_tokens: def cluster ( ctx , config ) : if ctx . config . get ( 'use_existing_cluster' , False ) is True : log . info ( "'use_existing_cluster' is true; skipping cluster creation" ) yield testdir = teuthology . get_testdir ( ctx ) cluster_name = config [ 'cluster' ] data_dir = '{tdir}/{cluster}.data' . format ( tdir = testdir , cluster = cluster_name ) log . info ( 'Creating ceph cluster %s...' , cluster_name ) log . info ( 'config %s' , config ) log . info ( 'ctx.config %s' , ctx . config ) run . wait ( ctx . cluster . run ( args = [ 'install' , '-d' , '-m0755' , '--' , data_dir , ] , wait = False , ) ) run . wait ( ctx . cluster . run ( args = [ 'sudo' , 'install' , '-d' , '-m0777' , '--' , '/var/run/ceph' , ] , wait = False , ) ) devs_to_clean = { } remote_to_roles_to_devs = { } osds = ctx . cluster . only ( teuthology . is_type ( 'osd' , cluster_name ) ) for remote , roles_for_host in osds . remotes . items ( ) : devs = teuthology . get_scratch_devices ( remote ) roles_to_devs = assign_devs ( teuthology . cluster_roles_of_type ( roles_for_host , 'osd' , cluster_name ) , devs ) devs_to_clean [ remote ] = [ ] log . info ( 'osd dev map: {}' . format ( roles_to_devs ) ) assert roles_to_devs , "remote {} has osd roles, but no osd devices were specified!" . format ( remote . hostname ) remote_to_roles_to_devs [ remote ] = roles_to_devs log . info ( "remote_to_roles_to_devs: {}" . format ( remote_to_roles_to_devs ) ) for osd_role , dev_name in remote_to_roles_to_devs . items ( ) : assert dev_name , "{} has no associated device!" . format ( osd_role ) log . info ( 'Generating config...' ) remotes_and_roles = ctx . cluster . remotes . items ( ) roles = [ role_list for ( remote , role_list ) in remotes_and_roles ] ips = [ host for ( host , port ) in ( remote . ssh . get_transport ( ) . getpeername ( ) for ( remote , role_list ) in remotes_and_roles ) ] mons = get_mons ( roles , ips , cluster_name , mon_bind_msgr2 = config . get ( 'mon_bind_msgr2' ) , mon_bind_addrvec = config . get ( 'mon_bind_addrvec' ) , ) conf = skeleton_config ( ctx , roles = roles , ips = ips , mons = mons , cluster = cluster_name , ) for section , keys in config [ 'conf' ] . items ( ) : for key , value in keys . items ( ) : log . info ( "[%s] %s = %s" % ( section , key , value ) ) if section not in conf : conf [ section ] = { } conf [ section ] [ key ] = value if not hasattr ( ctx , 'ceph' ) : ctx . ceph = { } ctx . ceph [ cluster_name ] = argparse . Namespace ( ) ctx . ceph [ cluster_name ] . conf = conf ctx . ceph [ cluster_name ] . mons = mons default_keyring = '/etc/ceph/{cluster}.keyring' . format ( cluster = cluster_name ) keyring_path = config . get ( 'keyring_path' , default_keyring ) coverage_dir = '{tdir}/archive/coverage' . format ( tdir = testdir ) firstmon = teuthology . get_first_mon ( ctx , config , cluster_name ) log . info ( 'Setting up %s...' % firstmon ) ctx . cluster . only ( firstmon ) . run ( args = [ 'sudo' , 'adjust-ulimits' , 'ceph-coverage' , coverage_dir , 'ceph-authtool' , '--create-keyring' , keyring_path , ] , ) ctx . cluster . only ( firstmon ) . run ( args = [ 'sudo' , 'adjust-ulimits' , 'ceph-coverage' , coverage_dir , 'ceph-authtool' , '--gen-key' , '--name=mon.' , keyring_path , ] , ) ctx . cluster . only ( firstmon ) . run ( args = [ 'sudo' , 'chmod' , '0644' , keyring_path , ] , ) ( mon0_remote , ) = ctx . cluster . only ( firstmon ) . remotes . keys ( ) monmap_path = '{tdir}/{cluster}.monmap' . format ( tdir = testdir , cluster = cluster_name ) fsid = create_simple_monmap ( ctx , remote = mon0_remote , conf = conf , mons = mons , path = monmap_path , mon_bind_addrvec = config . get ( 'mon_bind_addrvec' ) , ) ctx . ceph [ cluster_name ] . fsid = fsid if not 'global' in conf : conf [ 'global' ] = { } conf [ 'global' ] [ 'fsid' ] = fsid default_conf_path = '/etc/ceph/{cluster}.conf' . format ( cluster = cluster_name ) conf_path = config . get ( 'conf_path' , default_conf_path ) log . info ( 'Writing %s for FSID %s...' % ( conf_path , fsid ) ) write_conf ( ctx , conf_path , cluster_name ) log . info ( 'Creating admin key on %s...' % firstmon ) ctx . cluster . only ( firstmon ) . run ( args = [ 'sudo' , 'adjust-ulimits' , 'ceph-coverage' , coverage_dir , 'ceph-authtool' , '--gen-key' , '--name=client.admin' , '--cap' , 'mon' , 'allow *' , '--cap' , 'osd' , 'allow *' , '--cap' , 'mds' , 'allow *' , '--cap' , 'mgr' , 'allow *' , keyring_path , ] , ) log . info ( 'Copying monmap to all nodes...' ) keyring = mon0_remote . read_file ( keyring_path ) monmap = mon0_remote . read_file ( monmap_path ) for rem in ctx . cluster . remotes . keys ( ) : # copy mon key and initial monmap log . info ( 'Sending monmap to node {remote}' . format ( remote = rem ) ) rem . write_file ( keyring_path , keyring , mode = '0644' , sudo = True ) rem . write_file ( monmap_path , monmap ) log . info ( 'Setting up mon nodes...' ) mons = ctx . cluster . only ( teuthology . is_type ( 'mon' , cluster_name ) ) if not config . get ( 'skip_mgr_daemons' , False ) : log . info ( 'Setting up mgr nodes...' ) mgrs = ctx . cluster . only ( teuthology . is_type ( 'mgr' , cluster_name ) ) for remote , roles_for_host in mgrs . remotes . items ( ) : for role in teuthology . cluster_roles_of_type ( roles_for_host , 'mgr' , cluster_name ) : _ , _ , id_ = teuthology . split_role ( role ) mgr_dir = DATA_PATH . format ( type_ = 'mgr' , cluster = cluster_name , id_ = id_ ) remote . run ( args = [ 'sudo' , 'mkdir' , '-p' , mgr_dir , run . Raw ( '&&' ) , 'sudo' , 'adjust-ulimits' , 'ceph-coverage' , coverage_dir , 'ceph-authtool' , '--create-keyring' , '--gen-key' , '--name=mgr.{id}' . format ( id = id_ ) , mgr_dir + '/keyring' , ] , ) log . info ( 'Setting up mds nodes...' ) mdss = ctx . cluster . only ( teuthology . is_type ( 'mds' , cluster_name ) ) for remote , roles_for_host in mdss . remotes . items ( ) : for role in teuthology . cluster_roles_of_type ( roles_for_host , 'mds' , cluster_name ) : _ , _ , id_ = teuthology . split_role ( role ) mds_dir = DATA_PATH . format ( type_ = 'mds' , cluster = cluster_name , id_ = id_ ) remote . run ( args = [ 'sudo' , 'mkdir' , '-p' , mds_dir , run . Raw ( '&&' ) , 'sudo' , 'adjust-ulimits' , 'ceph-coverage' , coverage_dir , 'ceph-authtool' , '--create-keyring' , '--gen-key' , '--name=mds.{id}' . format ( id = id_ ) , mds_dir + '/keyring' , ] , ) remote . run ( args = [ 'sudo' , 'chown' , '-R' , 'ceph:ceph' , mds_dir ] ) cclient . create_keyring ( ctx , cluster_name ) log . info ( 'Running mkfs on osd nodes...' ) if not hasattr ( ctx , 'disk_config' ) : ctx . disk_config = argparse . Namespace ( ) if not hasattr ( ctx . disk_config , 'remote_to_roles_to_dev' ) : ctx . disk_config . remote_to_roles_to_dev = { } if not hasattr ( ctx . disk_config , 'remote_to_roles_to_dev_mount_options' ) : ctx . disk_config . remote_to_roles_to_dev_mount_options = { } if not hasattr ( ctx . disk_config , 'remote_to_roles_to_dev_fstype' ) : ctx . disk_config . remote_to_roles_to_dev_fstype = { } teuthology . deep_merge ( ctx . disk_config . remote_to_roles_to_dev , remote_to_roles_to_devs ) log . info ( "ctx.disk_config.remote_to_roles_to_dev: {r}" . format ( r = str ( ctx . disk_config . remote_to_roles_to_dev ) ) ) for remote , roles_for_host in osds . remotes . items ( ) : roles_to_devs = remote_to_roles_to_devs [ remote ] for role in teuthology . cluster_roles_of_type ( roles_for_host , 'osd' , cluster_name ) : _ , _ , id_ = teuthology . split_role ( role ) mnt_point = DATA_PATH . format ( type_ = 'osd' , cluster = cluster_name , id_ = id_ ) remote . run ( args = [ 'sudo' , 'mkdir' , '-p' , mnt_point , ] ) log . info ( 'roles_to_devs: {}' . format ( roles_to_devs ) ) log . info ( 'role: {}' . format ( role ) ) if roles_to_devs . get ( role ) : dev = roles_to_devs [ role ] fs = config . get ( 'fs' ) package = None mkfs_options = config . get ( 'mkfs_options' ) mount_options = config . get ( 'mount_options' ) if fs == 'btrfs' : # package = 'btrfs-tools' if mount_options is None : mount_options = [ 'noatime' , 'user_subvol_rm_allowed' ] if mkfs_options is None : mkfs_options = [ '-m' , 'single' , '-l' , '32768' , '-n' , '32768' ] if fs == 'xfs' : # package = 'xfsprogs' if mount_options is None : mount_options = [ 'noatime' ] if mkfs_options is None : mkfs_options = [ '-f' , '-i' , 'size=2048' ] if fs == 'ext4' or fs == 'ext3' : if mount_options is None : mount_options = [ 'noatime' , 'user_xattr' ] if mount_options is None : mount_options = [ ] if mkfs_options is None : mkfs_options = [ ] mkfs = [ 'mkfs.%s' % fs ] + mkfs_options log . info ( '%s on %s on %s' % ( mkfs , dev , remote ) ) if package is not None : remote . sh ( 'sudo apt-get install -y %s' % package ) try : remote . run ( args = [ 'yes' , run . Raw ( '|' ) ] + [ 'sudo' ] + mkfs + [ dev ] ) except run . CommandFailedError : # Newer btfs-tools doesn't prompt for overwrite, use -f if '-f' not in mount_options : mkfs_options . append ( '-f' ) mkfs = [ 'mkfs.%s' % fs ] + mkfs_options log . info ( '%s on %s on %s' % ( mkfs , dev , remote ) ) remote . run ( args = [ 'yes' , run . Raw ( '|' ) ] + [ 'sudo' ] + mkfs + [ dev ] ) log . info ( 'mount %s on %s -o %s' % ( dev , remote , ',' . join ( mount_options ) ) ) remote . run ( args = [ 'sudo' , 'mount' , '-t' , fs , '-o' , ',' . join ( mount_options ) , dev , mnt_point , ] ) remote . run ( args = [ 'sudo' , '/sbin/restorecon' , mnt_point , ] , check_status = False , ) if not remote in ctx . disk_config . remote_to_roles_to_dev_mount_options : ctx . disk_config . remote_to_roles_to_dev_mount_options [ remote ] = { } ctx . disk_config . remote_to_roles_to_dev_mount_options [ remote ] [ role ] = mount_options if not remote in ctx . disk_config . remote_to_roles_to_dev_fstype : ctx . disk_config . remote_to_roles_to_dev_fstype [ remote ] = { } ctx . disk_config . remote_to_roles_to_dev_fstype [ remote ] [ role ] = fs devs_to_clean [ remote ] . append ( mnt_point ) for role in teuthology . cluster_roles_of_type ( roles_for_host , 'osd' , cluster_name ) : _ , _ , id_ = teuthology . split_role ( role ) try : args = [ 'sudo' , 'MALLOC_CHECK_=3' , 'adjust-ulimits' , 'ceph-coverage' , coverage_dir , 'ceph-osd' , '--no-mon-config' , '--cluster' , cluster_name , '--mkfs' , '--mkkey' , '-i' , id_ , '--monmap' , monmap_path ] log_path = f'/var/log/ceph/{cluster_name}-osd.{id_}.log' create_log_cmd , args = maybe_redirect_stderr ( config , 'osd' , args , log_path ) if create_log_cmd : remote . sh ( create_log_cmd ) remote . run ( args = args ) except run . CommandFailedError : # try without --no-mon-config.. this may be an upgrade test remote . run ( args = [ 'sudo' , 'MALLOC_CHECK_=3' , 'adjust-ulimits' , 'ceph-coverage' , coverage_dir , 'ceph-osd' , '--cluster' , cluster_name , '--mkfs' , '--mkkey' , '-i' , id_ , '--monmap' , monmap_path , ] , ) mnt_point = DATA_PATH . format ( type_ = 'osd' , cluster = cluster_name , id_ = id_ ) remote . run ( args = [ 'sudo' , 'chown' , '-R' , 'ceph:ceph' , mnt_point ] ) log . info ( 'Reading keys from all nodes...' ) keys_fp = BytesIO ( ) keys = [ ] for remote , roles_for_host in ctx . cluster . remotes . items ( ) : for type_ in [ 'mgr' , 'mds' , 'osd' ] : if type_ == 'mgr' and config . get ( 'skip_mgr_daemons' , False ) : continue for role in teuthology . cluster_roles_of_type ( roles_for_host , type_ , cluster_name ) : _ , _ , id_ = teuthology . split_role ( role ) data = remote . read_file ( os . path . join ( DATA_PATH . format ( type_ = type_ , id_ = id_ , cluster = cluster_name ) , 'keyring' , ) , sudo = True , ) keys . append ( ( type_ , id_ , data ) ) keys_fp . write ( data ) for remote , roles_for_host in ctx . cluster . remotes . items ( ) : for role in teuthology . cluster_roles_of_type ( roles_for_host , 'client' , cluster_name ) : _ , _ , id_ = teuthology . split_role ( role ) data = remote . read_file ( '/etc/ceph/{cluster}.client.{id}.keyring' . format ( id = id_ , cluster = cluster_name ) ) keys . append ( ( 'client' , id_ , data ) ) keys_fp . write ( data ) log . info ( 'Adding keys to all mons...' ) writes = mons . run ( args = [ 'sudo' , 'tee' , '-a' , keyring_path , ] , stdin = run . PIPE , wait = False , stdout = BytesIO ( ) , ) keys_fp . seek ( 0 ) teuthology . feed_many_stdins_and_close ( keys_fp , writes ) run . wait ( writes ) for type_ , id_ , data in keys : run . wait ( mons . run ( args = [ 'sudo' , 'adjust-ulimits' , 'ceph-coverage' , coverage_dir , 'ceph-authtool' , keyring_path , '--name={type}.{id}' . format ( type = type_ , id = id_ , ) , ] + list ( generate_caps ( type_ ) ) , wait = False , ) , ) log . info ( 'Running mkfs on mon nodes...' ) for remote , roles_for_host in mons . remotes . items ( ) : for role in teuthology . cluster_roles_of_type ( roles_for_host , 'mon' , cluster_name ) : _ , _ , id_ = teuthology . split_role ( role ) mnt_point = DATA_PATH . format ( type_ = 'mon' , id_ = id_ , cluster = cluster_name ) remote . run ( args = [ 'sudo' , 'mkdir' , '-p' , mnt_point , ] , ) remote . run ( args = [ 'sudo' , 'adjust-ulimits' , 'ceph-coverage' , coverage_dir , 'ceph-mon' , '--cluster' , cluster_name , '--mkfs' , '-i' , id_ , '--monmap' , monmap_path , '--keyring' , keyring_path , ] , ) remote . run ( args = [ 'sudo' , 'chown' , '-R' , 'ceph:ceph' , mnt_point ] ) run . wait ( mons . run ( args = [ 'rm' , '--' , monmap_path , ] , wait = False , ) , ) try : yield except Exception : # we need to know this below ctx . summary [ 'success' ] = False raise finally : ( mon0_remote , ) = ctx . cluster . only ( firstmon ) . remotes . keys ( ) log . info ( 'Checking cluster log for badness...' ) def first_in_ceph_log ( pattern , excludes ) : """
              Find the first occurrence of the pattern specified in the Ceph log,
              Returns None if none found.
  
              :param pattern: Pattern scanned for.
              :param excludes: Patterns to ignore.
              :return: First line of text (or None if not found)
              """ args = [ 'sudo' , 'egrep' , pattern , '/var/log/ceph/{cluster}.log' . format ( cluster = cluster_name ) , ] for exclude in excludes : args . extend ( [ run . Raw ( '|' ) , 'egrep' , '-v' , exclude ] ) args . extend ( [ run . Raw ( '|' ) , 'head' , '-n' , '1' , ] ) stdout = mon0_remote . sh ( args ) return stdout or None if first_in_ceph_log ( '\[ERR\]|\[WRN\]|\[SEC\]' , config [ 'log_ignorelist' ] ) is not None : log . warning ( 'Found errors (ERR|WRN|SEC) in cluster log' ) ctx . summary [ 'success' ] = False # use the most severe problem as the failure reason if 'failure_reason' not in ctx . summary : for pattern in [ '\[SEC\]' , '\[ERR\]' , '\[WRN\]' ] : match = first_in_ceph_log ( pattern , config [ 'log_ignorelist' ] ) if match is not None : ctx . summary [ 'failure_reason' ] = '"{match}" in cluster log' . format ( match = match . rstrip ( '\n' ) , ) break for remote , dirs in devs_to_clean . items ( ) : for dir_ in dirs : log . info ( 'Unmounting %s on %s' % ( dir_ , remote ) ) try : remote . run ( args = [ 'sync' , run . Raw ( '&&' ) , 'sudo' , 'umount' , '-f' , dir_ ] ) except Exception as e : remote . run ( args = [ 'sudo' , run . Raw ( 'PATH=/usr/sbin:$PATH' ) , 'lsof' , run . Raw ( ';' ) , 'ps' , 'auxf' , ] ) raise e if ctx . archive is not None and not ( ctx . config . get ( 'archive-on-error' ) and ctx . summary [ 'success' ] ) : # archive mon data, too log . info ( 'Archiving mon data...' ) path = os . path . join ( ctx . archive , 'data' ) try : os . makedirs ( path ) except OSError as e : if e . errno == errno . EEXIST : pass else : raise for remote , roles in mons . remotes . items ( ) : for role in roles : is_mon = teuthology . is_type ( 'mon' , cluster_name ) if is_mon ( role ) : _ , _ , id_ = teuthology . split_role ( role ) mon_dir = DATA_PATH . format ( type_ = 'mon' , id_ = id_ , cluster = cluster_name ) teuthology . pull_directory_tarball ( remote , mon_dir , path + '/' + role + '.tgz' ) log . info ( 'Cleaning ceph cluster...' ) run . wait ( ctx . cluster . run ( args = [ 'sudo' , 'rm' , '-rf' , '--' , conf_path , keyring_path , data_dir , monmap_path , run . Raw ( '{tdir}/../*.pid' . format ( tdir = testdir ) ) , ] , wait = False , ) , )