Stop subprocesses from getting unexpectedly killed
The behaviour of `(syscall.SysProcAttr).Pdeathsig` on Linux has some surprising behaviour, as detailed in https://github.com/golang/go/issues/27505. The parent death signal is sent to the child process when the creating _thread_ dies, which in a Go program is going to be an arbitrary thread taken from the goroutine thread pool and recycled for other goroutines to execute on. A goroutine can cause the thread it is running on to exit by wiring itself to the thread with `runtime.LockOSThread()` and returning. Letting the creating thread become available for other goroutines risks having that thread terminated arbitrarily, and the child process with it.

While the problem can in theory be worked around by ensuring that no goroutine returns while wired to a thread, this is not practical. Unpaired `runtime.LockOSThread()` calls could be tucked away in third-party modules, and mistakes happen in first-party code. And there are legitimate use-cases for terminating threads early, which [heavily overlap with the use-cases for `pkg/reexec`](https://www.weave.works/blog/linux-namespaces-golang-followup), which makes it impractical to ban. The only viable solution is to arrange for the threads which are parents of subprocesses started with `Pdeathsig` to not be eligible for other goroutines to execute on them while the subprocess is alive.

**- What I did**
* Audited all calls to `runtime.LockOSThread()` in non-vendor code in the daemon process to ensure they had matching `runtime.UnlockOSThread()` calls. Unmatched calls in reexec functions were left alone.
* Ensured that the threads which create subprocesses with `Pdeathsig` could not be terminated before the subprocess or the daemon process exited, whichever comes first.

**- How I did it**
* `runtime.LockOSThread()` is both the problem and solution

**- How to verify it**
* CI 🟢 

**- Description for the changelog**
<!--
Write a short (one line) summary that describes the changes in this
pull request for inclusion in the changelog:
-->
* Resolved an issue which could potentially cause daemon-managed containerd or certain utility subprocesses from being prematurely killed on Linux

**- A picture of a cute animal (not mandatory but encouraged)**


pkg/chrootarchive/archive_unix.go
//go:build !windows
// +build !windows

package chrootarchive // import "github.com/docker/docker/pkg/chrootarchive"

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/reexec"
	"github.com/pkg/errors"
)

// untar is the entry-point for docker-untar on re-exec. This is not used on
// Windows as it does not support chroot, hence no point sandboxing through
// chroot and rexec.
func untar() {
	runtime.LockOSThread()
	flag.Parse()

	var options archive.TarOptions

	// read the options from the pipe "ExtraFiles"
	if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
		fatal(err)
	}

	dst := flag.Arg(0)
	var root string
	if len(flag.Args()) > 1 {
		root = flag.Arg(1)
	}

	if root == "" {
		root = dst
	}

	if err := chroot(root); err != nil {
		fatal(err)
	}

	if err := archive.Unpack(os.Stdin, dst, &options); err != nil {
		fatal(err)
	}
	// fully consume stdin in case it is zero padded
	if _, err := flush(os.Stdin); err != nil {
		fatal(err)
	}

	os.Exit(0)
}

func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions, root string) error {
	if root == "" {
		return errors.New("must specify a root to chroot to")
	}

	// We can't pass a potentially large exclude list directly via cmd line
	// because we easily overrun the kernel's max argument/environment size
	// when the full image list is passed (e.g. when this is used by
	// `docker load`). We will marshall the options via a pipe to the
	// child
	r, w, err := os.Pipe()
	if err != nil {
		return fmt.Errorf("Untar pipe failure: %v", err)
	}

	if root != "" {
		relDest, err := filepath.Rel(root, dest)
		if err != nil {
			return err
		}
		if relDest == "." {
			relDest = "/"
		}
		if relDest[0] != '/' {
			relDest = "/" + relDest
		}
		dest = relDest
	}

	cmd := reexec.Command("docker-untar", dest, root)
	cmd.Stdin = decompressedArchive

	cmd.ExtraFiles = append(cmd.ExtraFiles, r)
	output := bytes.NewBuffer(nil)
	cmd.Stdout = output
	cmd.Stderr = output

	if err := cmd.Start(); err != nil {
		w.Close()
		return fmt.Errorf("Untar error on re-exec cmd: %v", err)
	}

	// write the options to the pipe for the untar exec to read
	if err := json.NewEncoder(w).Encode(options); err != nil {
		w.Close()
		return fmt.Errorf("Untar json encode to pipe failed: %v", err)
	}
	w.Close()

	if err := cmd.Wait(); err != nil {
		// when `xz -d -c -q | docker-untar ...` failed on docker-untar side,
		// we need to exhaust `xz`'s output, otherwise the `xz` side will be
		// pending on write pipe forever
		io.Copy(io.Discard, decompressedArchive)

		return fmt.Errorf("Error processing tar file(%v): %s", err, output)
	}
	return nil
}

func tar() {
	runtime.LockOSThread()
	flag.Parse()

	src := flag.Arg(0)
	var root string
	if len(flag.Args()) > 1 {
		root = flag.Arg(1)
	}

	if root == "" {
		root = src
	}

	if err := realChroot(root); err != nil {
		fatal(err)
	}

	var options archive.TarOptions
	if err := json.NewDecoder(os.Stdin).Decode(&options); err != nil {
		fatal(err)
	}

	rdr, err := archive.TarWithOptions(src, &options)
	if err != nil {
		fatal(err)
	}
	defer rdr.Close()

	if _, err := io.Copy(os.Stdout, rdr); err != nil {
		fatal(err)
	}

	os.Exit(0)
}

func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) {
	if root == "" {
		return nil, errors.New("root path must not be empty")
	}

	relSrc, err := filepath.Rel(root, srcPath)
	if err != nil {
		return nil, err
	}
	if relSrc == "." {
		relSrc = "/"
	}
	if relSrc[0] != '/' {
		relSrc = "/" + relSrc
	}

	// make sure we didn't trim a trailing slash with the call to `Rel`
	if strings.HasSuffix(srcPath, "/") && !strings.HasSuffix(relSrc, "/") {
		relSrc += "/"
	}

	cmd := reexec.Command("docker-tar", relSrc, root)

	errBuff := bytes.NewBuffer(nil)
	cmd.Stderr = errBuff

	tarR, tarW := io.Pipe()
	cmd.Stdout = tarW

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, errors.Wrap(err, "error getting options pipe for tar process")
	}

	if err := cmd.Start(); err != nil {
		return nil, errors.Wrap(err, "tar error on re-exec cmd")
	}

	go func() {
		err := cmd.Wait()
		err = errors.Wrapf(err, "error processing tar file: %s", errBuff)
		tarW.CloseWithError(err)
	}()

	if err := json.NewEncoder(stdin).Encode(options); err != nil {
		stdin.Close()
		return nil, errors.Wrap(err, "tar json encode to pipe failed")
	}
	stdin.Close()

	return tarR, nil
}

//go:build !windows
// +build !windows

package chrootarchive // import "github.com/docker/docker/pkg/chrootarchive"

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/reexec"
	"github.com/pkg/errors"
)

// untar is the entry-point for docker-untar on re-exec. This is not used on
// Windows as it does not support chroot, hence no point sandboxing through
// chroot and rexec.
func untar() {
	runtime.LockOSThread()
	flag.Parse()

	var options archive.TarOptions

	// read the options from the pipe "ExtraFiles"
	if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
		fatal(err)
	}

	dst := flag.Arg(0)
	var root string
	if len(flag.Args()) > 1 {
		root = flag.Arg(1)
	}

	if root == "" {
		root = dst
	}

	if err := chroot(root); err != nil {
		fatal(err)
	}

	if err := archive.Unpack(os.Stdin, dst, &options); err != nil {
		fatal(err)
	}
	// fully consume stdin in case it is zero padded
	if _, err := flush(os.Stdin); err != nil {
		fatal(err)
	}

	os.Exit(0)
}

func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions, root string) error {
	if root == "" {
		return errors.New("must specify a root to chroot to")
	}

	// We can't pass a potentially large exclude list directly via cmd line
	// because we easily overrun the kernel's max argument/environment size
	// when the full image list is passed (e.g. when this is used by
	// `docker load`). We will marshall the options via a pipe to the
	// child
	r, w, err := os.Pipe()
	if err != nil {
		return fmt.Errorf("Untar pipe failure: %v", err)
	}

	if root != "" {
		relDest, err := filepath.Rel(root, dest)
		if err != nil {
			return err
		}
		if relDest == "." {
			relDest = "/"
		}
		if relDest[0] != '/' {
			relDest = "/" + relDest
		}
		dest = relDest
	}

	cmd := reexec.Command("docker-untar", dest, root)
	cmd.Stdin = decompressedArchive

	cmd.ExtraFiles = append(cmd.ExtraFiles, r)
	output := bytes.NewBuffer(nil)
	cmd.Stdout = output
	cmd.Stderr = output

	// reexec.Command() sets cmd.SysProcAttr.Pdeathsig on Linux, which
	// causes the started process to be signaled when the creating OS thread
	// dies. Ensure that the reexec is not prematurely signaled. See
	// https://go.dev/issue/27505 for more information.
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()
	if err := cmd.Start(); err != nil {
		w.Close()
		return fmt.Errorf("Untar error on re-exec cmd: %v", err)
	}

	// write the options to the pipe for the untar exec to read
	if err := json.NewEncoder(w).Encode(options); err != nil {
		w.Close()
		return fmt.Errorf("Untar json encode to pipe failed: %v", err)
	}
	w.Close()

	if err := cmd.Wait(); err != nil {
		// when `xz -d -c -q | docker-untar ...` failed on docker-untar side,
		// we need to exhaust `xz`'s output, otherwise the `xz` side will be
		// pending on write pipe forever
		io.Copy(io.Discard, decompressedArchive)

		return fmt.Errorf("Error processing tar file(%v): %s", err, output)
	}
	return nil
}

func tar() {
	runtime.LockOSThread()
	flag.Parse()

	src := flag.Arg(0)
	var root string
	if len(flag.Args()) > 1 {
		root = flag.Arg(1)
	}

	if root == "" {
		root = src
	}

	if err := realChroot(root); err != nil {
		fatal(err)
	}

	var options archive.TarOptions
	if err := json.NewDecoder(os.Stdin).Decode(&options); err != nil {
		fatal(err)
	}

	rdr, err := archive.TarWithOptions(src, &options)
	if err != nil {
		fatal(err)
	}
	defer rdr.Close()

	if _, err := io.Copy(os.Stdout, rdr); err != nil {
		fatal(err)
	}

	os.Exit(0)
}

func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) {
	if root == "" {
		return nil, errors.New("root path must not be empty")
	}

	relSrc, err := filepath.Rel(root, srcPath)
	if err != nil {
		return nil, err
	}
	if relSrc == "." {
		relSrc = "/"
	}
	if relSrc[0] != '/' {
		relSrc = "/" + relSrc
	}

	// make sure we didn't trim a trailing slash with the call to `Rel`
	if strings.HasSuffix(srcPath, "/") && !strings.HasSuffix(relSrc, "/") {
		relSrc += "/"
	}

	cmd := reexec.Command("docker-tar", relSrc, root)

	errBuff := bytes.NewBuffer(nil)
	cmd.Stderr = errBuff

	tarR, tarW := io.Pipe()
	cmd.Stdout = tarW

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, errors.Wrap(err, "error getting options pipe for tar process")
	}

	started := make(chan error)
	go func() {
		// reexec.Command() sets cmd.SysProcAttr.Pdeathsig on Linux,
		// which causes the started process to be signaled when the
		// creating OS thread dies. Ensure that the subprocess is not
		// prematurely signaled. See https://go.dev/issue/27505 for more
		// information.
		runtime.LockOSThread()
		defer runtime.UnlockOSThread()
		if err := cmd.Start(); err != nil {
			started <- err
			return
		}
		close(started)
		err := cmd.Wait()
		err = errors.Wrapf(err, "error processing tar file: %s", errBuff)
		tarW.CloseWithError(err)
	}()
	if err := <-started; err != nil {
		return nil, errors.Wrap(err, "tar error on re-exec cmd")
	}

	if err := json.NewEncoder(stdin).Encode(options); err != nil {
		stdin.Close()
		return nil, errors.Wrap(err, "tar json encode to pipe failed")
	}
	stdin.Close()

	return tarR, nil
}

Aha! That's the one! I'll fix that right up.