daemon: read-copy-update the daemon config
**- What I did**
Made daemon config reloading transactional and almost entirely atomic. Config reloading will either succeed or leave the daemon state untouched. Reloading no longer pauses the world. Daemon operations see a consistent and immutable view of the daemon config, even when racing a reload.

**- How I did it**
- Made the `config.Config` struct copyable by removing the embedded mutexes
- Implemented read-copy-update of the daemon config to replace the embedded mutex for synchronization
  - Reading from daemon config is request-scoped, like contexts: any operations which need to access the daemon config capture a reference to an immutable snapshot and pass it into any callees
- Implemented two-phase commit to manage the side effects of config reloading
  - Side effects such as `registryService.ReplaceConfig()` are not necessarily atomic w.r.t. the reload transaction as they can take effect before the updated config is applied to `configStore`
- Modified how runtimes wrapper scripts are written so that existing wrapper scripts are no longer deleted on reload
  - reloads do not invalidate old versions of the config that concurrent daemon operations might be holding references to, which might reference old wrapper scripts
  - FS errors while updating the wrapper scripts can no longer leave containers or the daemon in a bad state

**- How to verify it**
CI

**- Description for the changelog**
<!--
Write a short (one line) summary that describes the changes in this
pull request for inclusion in the changelog:
-->
- Configuration reloading has been made more robust: in the event that an error is encountered during the reload process, there is no longer a chance for some of the options to be applied anyway.

**- A picture of a cute animal (not mandatory but encouraged)**


daemon/runtime_unix.go
//go:build !windows

package daemon

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/daemon/config"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/libcontainerd/shimopts"
	"github.com/opencontainers/runtime-spec/specs-go/features"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	defaultRuntimeName = "runc"

	linuxShimV2 = "io.containerd.runc.v2"
)

func configureRuntimes(conf *config.Config) {
	if conf.DefaultRuntime == "" {
		conf.DefaultRuntime = config.StockRuntimeName
	}
	if conf.Runtimes == nil {
		conf.Runtimes = make(map[string]types.Runtime)
	}
	conf.Runtimes[config.LinuxV2RuntimeName] = types.Runtime{Path: defaultRuntimeName, ShimConfig: defaultV2ShimConfig(conf, defaultRuntimeName)}
	conf.Runtimes[config.StockRuntimeName] = conf.Runtimes[config.LinuxV2RuntimeName]
}

func defaultV2ShimConfig(conf *config.Config, runtimePath string) *types.ShimConfig {
	return &types.ShimConfig{
		Binary: linuxShimV2,
		Opts: &v2runcoptions.Options{
			BinaryName:    runtimePath,
			Root:          filepath.Join(conf.ExecRoot, "runtime-"+defaultRuntimeName),
			SystemdCgroup: UsingSystemd(conf),
			NoPivotRoot:   os.Getenv("DOCKER_RAMDISK") != "",
		},
	}
}

func (daemon *Daemon) loadRuntimes() error {
	return daemon.initRuntimes(daemon.configStore.Runtimes)
}

func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error) {
	runtimeDir := filepath.Join(daemon.configStore.Root, "runtimes")
	runtimeOldDir := runtimeDir + "-old"
	// Remove old temp directory if any
	os.RemoveAll(runtimeOldDir)
	tmpDir, err := os.MkdirTemp(daemon.configStore.Root, "gen-runtimes")
	if err != nil {
		return errors.Wrap(err, "failed to get temp dir to generate runtime scripts")
	}
	defer func() {
		if err != nil {
			if err1 := os.RemoveAll(tmpDir); err1 != nil {
				logrus.WithError(err1).WithField("dir", tmpDir).
					Warn("failed to remove tmp dir")
			}
			return
		}

		if err = os.Rename(runtimeDir, runtimeOldDir); err != nil {
			logrus.WithError(err).WithField("dir", runtimeDir).
				Warn("failed to rename runtimes dir to old. Will try to removing it")
			if err = os.RemoveAll(runtimeDir); err != nil {
				logrus.WithError(err).WithField("dir", runtimeDir).
					Warn("failed to remove old runtimes dir")
				return
			}
		}
		if err = os.Rename(tmpDir, runtimeDir); err != nil {
			err = errors.Wrap(err, "failed to setup runtimes dir, new containers may not start")
			return
		}
		if err = os.RemoveAll(runtimeOldDir); err != nil {
			logrus.WithError(err).WithField("dir", runtimeOldDir).
				Warn("failed to remove old runtimes dir")
		}
	}()

	for name := range runtimes {
		rt := runtimes[name]
		if rt.Path == "" && rt.Type == "" {
			return errors.Errorf("runtime %s: either a runtimeType or a path must be configured", name)
		}
		if rt.Path != "" {
			if rt.Type != "" {
				return errors.Errorf("runtime %s: cannot configure both path and runtimeType for the same runtime", name)
			}
			if len(rt.Options) > 0 {
				return errors.Errorf("runtime %s: options cannot be used with a path runtime", name)
			}

			if len(rt.Args) > 0 {
				script := filepath.Join(tmpDir, name)
				content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " "))
				if err := os.WriteFile(script, []byte(content), 0700); err != nil {
					return err
				}
			}
			rt.ShimConfig = defaultV2ShimConfig(daemon.configStore, daemon.rewriteRuntimePath(name, rt.Path, rt.Args))
			var featuresStderr bytes.Buffer
			featuresCmd := exec.Command(rt.Path, append(rt.Args, "features")...)
			featuresCmd.Stderr = &featuresStderr
			if featuresB, err := featuresCmd.Output(); err != nil {
				logrus.WithError(err).Warnf("Failed to run %v: %q", featuresCmd.Args, featuresStderr.String())
			} else {
				var features features.Features
				if jsonErr := json.Unmarshal(featuresB, &features); jsonErr != nil {
					logrus.WithError(err).Warnf("Failed to unmarshal the output of %v as a JSON", featuresCmd.Args)
				} else {
					rt.Features = &features
				}
			}
		} else {
			if len(rt.Args) > 0 {
				return errors.Errorf("runtime %s: args cannot be used with a runtimeType runtime", name)
			}
			// Unlike implicit runtimes, there is no restriction on configuring a shim by path.
			rt.ShimConfig = &types.ShimConfig{Binary: rt.Type}
			if len(rt.Options) > 0 {
				// It has to be a pointer type or there'll be a panic in containerd/typeurl when we try to start the container.
				rt.ShimConfig.Opts, err = shimopts.Generate(rt.Type, rt.Options)
				if err != nil {
					return errors.Wrapf(err, "runtime %v", name)
				}
			}
		}
		runtimes[name] = rt
	}
	return nil
}

// rewriteRuntimePath is used for runtimes which have custom arguments supplied.
// This is needed because the containerd API only calls the OCI runtime binary, there is no options for extra arguments.
// To support this case, the daemon wraps the specified runtime in a script that passes through those arguments.
func (daemon *Daemon) rewriteRuntimePath(name, p string, args []string) string {
	if len(args) == 0 {
		return p
	}

	return filepath.Join(daemon.configStore.Root, "runtimes", name)
}

func (daemon *Daemon) getRuntime(name string) (shim string, opts interface{}, err error) {
	rt := daemon.configStore.GetRuntime(name)
	if rt == nil {
		if !config.IsPermissibleC8dRuntimeName(name) {
			return "", nil, errdefs.InvalidParameter(errors.Errorf("unknown or invalid runtime name: %s", name))
		}
		return name, nil, nil
	}

	if len(rt.Args) > 0 {
		// Check that the path of the runtime which the script wraps actually exists so
		// that we can return a well known error which references the configured path
		// instead of the wrapper script's.
		if _, err := exec.LookPath(rt.Path); err != nil {
			return "", nil, errors.Wrap(err, "error while looking up the specified runtime path")
		}
	}

	if rt.ShimConfig == nil {
		// Should never happen as daemon.initRuntimes always sets
		// ShimConfig and config reloading is synchronized.
		err := errdefs.System(errors.Errorf("BUG: runtime %s: rt.ShimConfig == nil", name))
		logrus.Error(err)
		return "", nil, err
	}

	return rt.ShimConfig.Binary, rt.ShimConfig.Opts, nil
}

//go:build !windows

package daemon

import (
	"bytes"
	"crypto/sha256"
	"encoding/base32"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/containerd/containerd/plugin"
	v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
	"github.com/containerd/containerd/runtime/v2/shim"
	"github.com/docker/docker/daemon/config"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/libcontainerd/shimopts"
	"github.com/docker/docker/pkg/ioutils"
	"github.com/docker/docker/pkg/system"
	"github.com/opencontainers/runtime-spec/specs-go/features"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	defaultRuntimeName = "runc"

	// The runtime used to specify the containerd v2 runc shim
	linuxV2RuntimeName = "io.containerd.runc.v2"
)

type shimConfig struct {
	Shim     string
	Opts     interface{}
	Features *features.Features

	// Check if the ShimConfig is valid given the current state of the system.
	PreflightCheck func() error
}

type runtimes struct {
	Default    string
	configured map[string]*shimConfig
}

func stockRuntimes() map[string]string {
	return map[string]string{
		linuxV2RuntimeName:      defaultRuntimeName,
		config.StockRuntimeName: defaultRuntimeName,
	}
}

func defaultV2ShimConfig(conf *config.Config, runtimePath string) *shimConfig {
	shim := &shimConfig{
		Shim: plugin.RuntimeRuncV2,
		Opts: &v2runcoptions.Options{
			BinaryName:    runtimePath,
			Root:          filepath.Join(conf.ExecRoot, "runtime-"+defaultRuntimeName),
			SystemdCgroup: UsingSystemd(conf),
			NoPivotRoot:   os.Getenv("DOCKER_RAMDISK") != "",
		},
	}

	var featuresStderr bytes.Buffer
	featuresCmd := exec.Command(runtimePath, "features")
	featuresCmd.Stderr = &featuresStderr
	if featuresB, err := featuresCmd.Output(); err != nil {
		logrus.WithError(err).Warnf("Failed to run %v: %q", featuresCmd.Args, featuresStderr.String())
	} else {
		var features features.Features
		if jsonErr := json.Unmarshal(featuresB, &features); jsonErr != nil {
			logrus.WithError(err).Warnf("Failed to unmarshal the output of %v as a JSON", featuresCmd.Args)
		} else {
			shim.Features = &features
		}
	}

	return shim
}

func runtimeScriptsDir(cfg *config.Config) string {
	return filepath.Join(cfg.Root, "runtimes")
}

// initRuntimesDir creates a fresh directory where we'll store the runtime
// scripts (i.e. in order to support runtimeArgs).
func initRuntimesDir(cfg *config.Config) error {
	runtimeDir := runtimeScriptsDir(cfg)
	if err := os.RemoveAll(runtimeDir); err != nil {
		return err
	}
	return system.MkdirAll(runtimeDir, 0700)
}

func setupRuntimes(cfg *config.Config) (runtimes, error) {
	if _, ok := cfg.Runtimes[config.StockRuntimeName]; ok {
		return runtimes{}, errors.Errorf("runtime name '%s' is reserved", config.StockRuntimeName)
	}

	newrt := runtimes{
		Default:    cfg.DefaultRuntime,
		configured: make(map[string]*shimConfig),
	}
	for name, path := range stockRuntimes() {
		newrt.configured[name] = defaultV2ShimConfig(cfg, path)
	}

	if newrt.Default != "" {
		_, isStock := newrt.configured[newrt.Default]
		_, isConfigured := cfg.Runtimes[newrt.Default]
		if !isStock && !isConfigured && !isPermissibleC8dRuntimeName(newrt.Default) {
			return runtimes{}, errors.Errorf("specified default runtime '%s' does not exist", newrt.Default)
		}
	} else {
		newrt.Default = config.StockRuntimeName
	}

	dir := runtimeScriptsDir(cfg)
	for name, rt := range cfg.Runtimes {
		var c *shimConfig
		if rt.Path == "" && rt.Type == "" {
			return runtimes{}, errors.Errorf("runtime %s: either a runtimeType or a path must be configured", name)
		}
		if rt.Path != "" {
			if rt.Type != "" {
				return runtimes{}, errors.Errorf("runtime %s: cannot configure both path and runtimeType for the same runtime", name)
			}
			if len(rt.Options) > 0 {
				return runtimes{}, errors.Errorf("runtime %s: options cannot be used with a path runtime", name)
			}

			binaryName := rt.Path
			needsWrapper := len(rt.Args) > 0
			if needsWrapper {
				var err error
				binaryName, err = wrapRuntime(dir, name, rt.Path, rt.Args)
				if err != nil {
					return runtimes{}, err
				}
			}
			c = defaultV2ShimConfig(cfg, binaryName)
			if needsWrapper {
				path := rt.Path
				c.PreflightCheck = func() error {
					// Check that the runtime path actually exists so that we can return a well known error.
					_, err := exec.LookPath(path)
					return errors.Wrap(err, "error while looking up the specified runtime path")
				}
			}
		} else {
			if len(rt.Args) > 0 {
				return runtimes{}, errors.Errorf("runtime %s: args cannot be used with a runtimeType runtime", name)
			}
			// Unlike implicit runtimes, there is no restriction on configuring a shim by path.
			c = &shimConfig{Shim: rt.Type}
			if len(rt.Options) > 0 {
				// It has to be a pointer type or there'll be a panic in containerd/typeurl when we try to start the container.
				var err error
				c.Opts, err = shimopts.Generate(rt.Type, rt.Options)
				if err != nil {
					return runtimes{}, errors.Wrapf(err, "runtime %v", name)
				}
			}
		}
		newrt.configured[name] = c
	}

	return newrt, nil
}

// A non-standard Base32 encoding which lacks vowels to avoid accidentally
// spelling naughty words. Don't use this to encode any data which requires
// compatibility with anything outside of the currently-running process.
var base32Disemvoweled = base32.NewEncoding("0123456789BCDFGHJKLMNPQRSTVWXYZ-")

// wrapRuntime writes a shell script to dir which will execute binary with args
// concatenated to the script's argv. This is needed because the
// io.containerd.runc.v2 shim has no options for passing extra arguments to the
// runtime binary.
func wrapRuntime(dir, name, binary string, args []string) (string, error) {
	var wrapper bytes.Buffer
	sum := sha256.New()
	_, _ = fmt.Fprintf(io.MultiWriter(&wrapper, sum), "#!/bin/sh\n%s %s $@\n", binary, strings.Join(args, " "))
	// Generate a consistent name for the wrapper script derived from the
	// contents so that multiple wrapper scripts can coexist with the same
	// base name. The existing scripts might still be referenced by running
	// containers.
	suffix := base32Disemvoweled.EncodeToString(sum.Sum(nil))
	scriptPath := filepath.Join(dir, name+"."+suffix)
	if err := ioutils.AtomicWriteFile(scriptPath, wrapper.Bytes(), 0700); err != nil {
		return "", err
	}
	return scriptPath, nil
}

// Get returns the containerd runtime and options for name, suitable to pass
// into containerd.WithRuntime(). The runtime and options for the default
// runtime are returned when name is the empty string.
func (r *runtimes) Get(name string) (string, interface{}, error) {
	if name == "" {
		name = r.Default
	}

	rt := r.configured[name]
	if rt != nil {
		if rt.PreflightCheck != nil {
			if err := rt.PreflightCheck(); err != nil {
				return "", nil, err
			}
		}
		return rt.Shim, rt.Opts, nil
	}

	if !isPermissibleC8dRuntimeName(name) {
		return "", nil, errdefs.InvalidParameter(errors.Errorf("unknown or invalid runtime name: %s", name))
	}
	return name, nil, nil
}

func (r *runtimes) Features(name string) *features.Features {
	if name == "" {
		name = r.Default
	}

	rt := r.configured[name]
	if rt != nil {
		return rt.Features
	}
	return nil
}

// isPermissibleC8dRuntimeName tests whether name is safe to pass into
// containerd as a runtime name, and whether the name is well-formed.
// It does not check if the runtime is installed.
//
// A runtime name containing slash characters is interpreted by containerd as
// the path to a runtime binary. If we allowed this, anyone with Engine API
// access could get containerd to execute an arbitrary binary as root. Although
// Engine API access is already equivalent to root on the host, the runtime name
// has not historically been a vector to run arbitrary code as root so users are
// not expecting it to become one.
//
// This restriction is not configurable. There are viable workarounds for
// legitimate use cases: administrators and runtime developers can make runtimes
// available for use with Docker by installing them onto PATH following the
// [binary naming convention] for containerd Runtime v2.
//
// [binary naming convention]: https://github.com/containerd/containerd/blob/main/runtime/v2/README.md#binary-naming
func isPermissibleC8dRuntimeName(name string) bool {
	// containerd uses a rather permissive test to validate runtime names:
	//
	//   - Any name for which filepath.IsAbs(name) is interpreted as the absolute
	//     path to a shim binary. We want to block this behaviour.
	//   - Any name which contains at least one '.' character and no '/' characters
	//     and does not begin with a '.' character is a valid runtime name. The shim
	//     binary name is derived from the final two components of the name and
	//     searched for on the PATH. The name "a.." is technically valid per
	//     containerd's implementation: it would resolve to a binary named
	//     "containerd-shim---".
	//
	// https://github.com/containerd/containerd/blob/11ded166c15f92450958078cd13c6d87131ec563/runtime/v2/manager.go#L297-L317
	// https://github.com/containerd/containerd/blob/11ded166c15f92450958078cd13c6d87131ec563/runtime/v2/shim/util.go#L83-L93
	return !filepath.IsAbs(name) && !strings.ContainsRune(name, '/') && shim.BinaryName(name) != ""
}

:joy: :heart:

(Should this also remove "vowel looking" numbers like `0`, `1`, `3`, `4`?)