state/Wait: Fix race when reading exit status
Fixes:
- https://github.com/moby/moby/issues/39352

Before this change there was a race condition between State.Wait reading
the exit code from State and the State being changed immediately after the
change which ended the State.Wait.

Now, each State.Wait call holds a pointer to the StateStatus obtained
from State at the time of State.Wait call. This makes it possible to
read the correct exit code, even if the State was already modified.

The issue caused the `docker run --restart=always ...' to sometimes exit
with 0 exit code, because the process was already restarted by the time
State.Wait got the chance to read the exit code.

Test run
Before:
```
$ go test -count 1 -run TestCorrectStateWaitResultAfterRestart .
--- FAIL: TestCorrectStateWaitResultAfterRestart (0.00s)
    state_test.go:198: expected exit code 10, got 0
FAIL
FAIL    github.com/docker/docker/container      0.011s
FAIL

```

After:
```
$ go test -count 1 -run TestCorrectStateWaitResultAfterRestart .
ok      github.com/docker/docker/container      0.011s
```

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>

**- What I did**
Fixed State.Wait returning unexpected StateStatus

**- How I did it**
Store StateStatus for Wait and make each Wait's goroutine store a pointer to it. The subsequent State modifications will allocate a new StateStatus, so the old one won't be affected. 

**- How to verify it**
Run tests:
```
cd container
go test -count 1 -run TestCorrectStateWaitResultAfterRestart .
```
or

(it's faster if you run multiple of those with different names)
```
while [ true ]; do
  docker run --name test --restart=always busybox sh -c 'exit 11'
  if [ ! $? -eq 11 ]; then
    echo "Reproduced!"
    break
   fi
   docker rm -f test
done
```
**- Description for the changelog**
Fix `docker run` with restart policy to sometimes exit with 0 code

**- A picture of a cute animal (not mandatory but encouraged)**
![gimmebananahuman](https://user-images.githubusercontent.com/5046555/176409011-6cdc8460-6c4b-4008-802e-a35c406f8991.jpg)


daemon/restart.go
package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"fmt"

	containertypes "github.com/docker/docker/api/types/container"
	"github.com/docker/docker/container"
	"github.com/sirupsen/logrus"
)

// ContainerRestart stops and starts a container. It attempts to
// gracefully stop the container within the given timeout, forcefully
// stopping it if the timeout is exceeded. If given a negative
// timeout, ContainerRestart will wait forever until a graceful
// stop. Returns an error if the container cannot be found, or if
// there is an underlying error at any stage of the restart.
func (daemon *Daemon) ContainerRestart(ctx context.Context, name string, options containertypes.StopOptions) error {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}
	err = daemon.containerRestart(ctx, ctr, options)
	if err != nil {
		return fmt.Errorf("Cannot restart container %s: %v", name, err)
	}
	return nil

}

// containerRestart attempts to gracefully stop and then start the
// container. When stopping, wait for the given duration in seconds to
// gracefully stop, before forcefully terminating the container. If
// given a negative duration, wait forever for a graceful stop.
func (daemon *Daemon) containerRestart(ctx context.Context, container *container.Container, options containertypes.StopOptions) error {
	// Determine isolation. If not specified in the hostconfig, use daemon default.
	actualIsolation := container.HostConfig.Isolation
	if containertypes.Isolation.IsDefault(actualIsolation) {
		actualIsolation = daemon.defaultIsolation
	}

	// Avoid unnecessarily unmounting and then directly mounting
	// the container when the container stops and then starts
	// again. We do not do this for Hyper-V isolated containers
	// (implying also on Windows) as the HCS must have exclusive
	// access to mount the containers filesystem inside the utility
	// VM.
	if !containertypes.Isolation.IsHyperV(actualIsolation) {
		if err := daemon.Mount(container); err == nil {
			defer daemon.Unmount(container)
		}
	}

	if container.IsRunning() {
		// set AutoRemove flag to false before stop so the container won't be
		// removed during restart process
		autoRemove := container.HostConfig.AutoRemove

		container.HostConfig.AutoRemove = false
		err := daemon.containerStop(ctx, container, options)
		// restore AutoRemove irrespective of whether the stop worked or not
		container.HostConfig.AutoRemove = autoRemove
		// containerStop will write HostConfig to disk, we shall restore AutoRemove
		// in disk too
		if toDiskErr := daemon.checkpointAndSave(container); toDiskErr != nil {
			logrus.Errorf("Write container to disk error: %v", toDiskErr)
		}

		if err != nil {
			return err
		}
	}

	if err := daemon.containerStart(container, "", "", true); err != nil {
		return err
	}

	daemon.LogContainerEvent(container, "restart")
	return nil
}

package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"fmt"

	containertypes "github.com/docker/docker/api/types/container"
	"github.com/docker/docker/container"
)

// ContainerRestart stops and starts a container. It attempts to
// gracefully stop the container within the given timeout, forcefully
// stopping it if the timeout is exceeded. If given a negative
// timeout, ContainerRestart will wait forever until a graceful
// stop. Returns an error if the container cannot be found, or if
// there is an underlying error at any stage of the restart.
func (daemon *Daemon) ContainerRestart(ctx context.Context, name string, options containertypes.StopOptions) error {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}
	err = daemon.containerRestart(ctx, ctr, options)
	if err != nil {
		return fmt.Errorf("Cannot restart container %s: %v", name, err)
	}
	return nil

}

// containerRestart attempts to gracefully stop and then start the
// container. When stopping, wait for the given duration in seconds to
// gracefully stop, before forcefully terminating the container. If
// given a negative duration, wait forever for a graceful stop.
func (daemon *Daemon) containerRestart(ctx context.Context, container *container.Container, options containertypes.StopOptions) error {
	// Determine isolation. If not specified in the hostconfig, use daemon default.
	actualIsolation := container.HostConfig.Isolation
	if containertypes.Isolation.IsDefault(actualIsolation) {
		actualIsolation = daemon.defaultIsolation
	}

	// Avoid unnecessarily unmounting and then directly mounting
	// the container when the container stops and then starts
	// again. We do not do this for Hyper-V isolated containers
	// (implying also on Windows) as the HCS must have exclusive
	// access to mount the containers filesystem inside the utility
	// VM.
	if !containertypes.Isolation.IsHyperV(actualIsolation) {
		if err := daemon.Mount(container); err == nil {
			defer daemon.Unmount(container)
		}
	}

	if container.IsRunning() {
		container.Lock()
		container.HasBeenManuallyRestarted = true
		container.Unlock()

		err := daemon.containerStop(ctx, container, options)

		if err != nil {
			return err
		}
	}

	if err := daemon.containerStart(container, "", "", true); err != nil {
		return err
	}

	daemon.LogContainerEvent(container, "restart")
	return nil
}

If we call this in a `defer`, it will be called after `daemon.checkpointAndSave`, which I think will mean that the container configuration (on disk) would now always have `AutoRemove: false` 🤔 