state/Wait: Fix race when reading exit status
Fixes:
- https://github.com/moby/moby/issues/39352

Before this change there was a race condition between State.Wait reading
the exit code from State and the State being changed immediately after the
change which ended the State.Wait.

Now, each State.Wait call holds a pointer to the StateStatus obtained
from State at the time of State.Wait call. This makes it possible to
read the correct exit code, even if the State was already modified.

The issue caused the `docker run --restart=always ...' to sometimes exit
with 0 exit code, because the process was already restarted by the time
State.Wait got the chance to read the exit code.

Test run
Before:
```
$ go test -count 1 -run TestCorrectStateWaitResultAfterRestart .
--- FAIL: TestCorrectStateWaitResultAfterRestart (0.00s)
    state_test.go:198: expected exit code 10, got 0
FAIL
FAIL    github.com/docker/docker/container      0.011s
FAIL

```

After:
```
$ go test -count 1 -run TestCorrectStateWaitResultAfterRestart .
ok      github.com/docker/docker/container      0.011s
```

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>

**- What I did**
Fixed State.Wait returning unexpected StateStatus

**- How I did it**
Store StateStatus for Wait and make each Wait's goroutine store a pointer to it. The subsequent State modifications will allocate a new StateStatus, so the old one won't be affected. 

**- How to verify it**
Run tests:
```
cd container
go test -count 1 -run TestCorrectStateWaitResultAfterRestart .
```
or

(it's faster if you run multiple of those with different names)
```
while [ true ]; do
  docker run --name test --restart=always busybox sh -c 'exit 11'
  if [ ! $? -eq 11 ]; then
    echo "Reproduced!"
    break
   fi
   docker rm -f test
done
```
**- Description for the changelog**
Fix `docker run` with restart policy to sometimes exit with 0 code

**- A picture of a cute animal (not mandatory but encouraged)**
![gimmebananahuman](https://user-images.githubusercontent.com/5046555/176409011-6cdc8460-6c4b-4008-802e-a35c406f8991.jpg)


container/state_test.go
package container // import "github.com/docker/docker/container"

import (
	"context"
	"testing"
	"time"

	"github.com/docker/docker/api/types"
)

func TestIsValidHealthString(t *testing.T) {
	contexts := []struct {
		Health   string
		Expected bool
	}{
		{types.Healthy, true},
		{types.Unhealthy, true},
		{types.Starting, true},
		{types.NoHealthcheck, true},
		{"fail", false},
	}

	for _, c := range contexts {
		v := IsValidHealthString(c.Health)
		if v != c.Expected {
			t.Fatalf("Expected %t, but got %t", c.Expected, v)
		}
	}
}

func TestStateRunStop(t *testing.T) {
	s := NewState()

	// Begin another wait with WaitConditionRemoved. It should complete
	// within 200 milliseconds.
	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
	defer cancel()
	removalWait := s.Wait(ctx, WaitConditionRemoved)

	// Full lifecycle two times.
	for i := 1; i <= 2; i++ {
		// A wait with WaitConditionNotRunning should return
		// immediately since the state is now either "created" (on the
		// first iteration) or "exited" (on the second iteration). It
		// shouldn't take more than 50 milliseconds.
		ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
		defer cancel()
		// Expectx exit code to be i-1 since it should be the exit
		// code from the previous loop or 0 for the created state.
		if status := <-s.Wait(ctx, WaitConditionNotRunning); status.ExitCode() != i-1 {
			t.Fatalf("ExitCode %v, expected %v, err %q", status.ExitCode(), i-1, status.Err())
		}

		// A wait with WaitConditionNextExit should block until the
		// container has started and exited. It shouldn't take more
		// than 100 milliseconds.
		ctx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		initialWait := s.Wait(ctx, WaitConditionNextExit)

		// Set the state to "Running".
		s.Lock()
		s.SetRunning(i, true)
		s.Unlock()

		// Assert desired state.
		if !s.IsRunning() {
			t.Fatal("State not running")
		}
		if s.Pid != i {
			t.Fatalf("Pid %v, expected %v", s.Pid, i)
		}
		if s.ExitCode() != 0 {
			t.Fatalf("ExitCode %v, expected 0", s.ExitCode())
		}

		// Now that it's running, a wait with WaitConditionNotRunning
		// should block until we stop the container. It shouldn't take
		// more than 100 milliseconds.
		ctx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		exitWait := s.Wait(ctx, WaitConditionNotRunning)

		// Set the state to "Exited".
		s.Lock()
		s.SetStopped(&ExitStatus{ExitCode: i})
		s.Unlock()

		// Assert desired state.
		if s.IsRunning() {
			t.Fatal("State is running")
		}
		if s.ExitCode() != i {
			t.Fatalf("ExitCode %v, expected %v", s.ExitCode(), i)
		}
		if s.Pid != 0 {
			t.Fatalf("Pid %v, expected 0", s.Pid)
		}

		// Receive the initialWait result.
		if status := <-initialWait; status.ExitCode() != i {
			t.Fatalf("ExitCode %v, expected %v, err %q", status.ExitCode(), i, status.Err())
		}

		// Receive the exitWait result.
		if status := <-exitWait; status.ExitCode() != i {
			t.Fatalf("ExitCode %v, expected %v, err %q", status.ExitCode(), i, status.Err())
		}
	}

	// Set the state to dead and removed.
	s.Lock()
	s.Dead = true
	s.Unlock()
	s.SetRemoved()

	// Wait for removed status or timeout.
	if status := <-removalWait; status.ExitCode() != 2 {
		// Should have the final exit code from the loop.
		t.Fatalf("Removal wait exitCode %v, expected %v, err %q", status.ExitCode(), 2, status.Err())
	}
}

func TestStateTimeoutWait(t *testing.T) {
	s := NewState()

	s.Lock()
	s.SetRunning(0, true)
	s.Unlock()

	// Start a wait with a timeout.
	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
	defer cancel()
	waitC := s.Wait(ctx, WaitConditionNotRunning)

	// It should timeout *before* this 200ms timer does.
	select {
	case <-time.After(200 * time.Millisecond):
		t.Fatal("Stop callback doesn't fire in 200 milliseconds")
	case status := <-waitC:
		t.Log("Stop callback fired")
		// Should be a timeout error.
		if status.Err() == nil {
			t.Fatal("expected timeout error, got nil")
		}
		if status.ExitCode() != -1 {
			t.Fatalf("expected exit code %v, got %v", -1, status.ExitCode())
		}
	}

	s.Lock()
	s.SetStopped(&ExitStatus{ExitCode: 0})
	s.Unlock()

	// Start another wait with a timeout. This one should return
	// immediately.
	ctx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond)
	defer cancel()
	waitC = s.Wait(ctx, WaitConditionNotRunning)

	select {
	case <-time.After(200 * time.Millisecond):
		t.Fatal("Stop callback doesn't fire in 200 milliseconds")
	case status := <-waitC:
		t.Log("Stop callback fired")
		if status.ExitCode() != 0 {
			t.Fatalf("expected exit code %v, got %v, err %q", 0, status.ExitCode(), status.Err())
		}
	}
}

func TestIsValidStateString(t *testing.T) {
	states := []struct {
		state    string
		expected bool
	}{
		{"paused", true},
		{"restarting", true},
		{"running", true},
		{"dead", true},
		{"start", false},
		{"created", true},
		{"exited", true},
		{"removing", true},
		{"stop", false},
	}

	for _, s := range states {
		v := IsValidStateString(s.state)
		if v != s.expected {
			t.Fatalf("Expected %t, but got %t", s.expected, v)
		}
	}
}

package container // import "github.com/docker/docker/container"

import (
	"context"
	"testing"
	"time"

	"github.com/docker/docker/api/types"
)

func TestIsValidHealthString(t *testing.T) {
	contexts := []struct {
		Health   string
		Expected bool
	}{
		{types.Healthy, true},
		{types.Unhealthy, true},
		{types.Starting, true},
		{types.NoHealthcheck, true},
		{"fail", false},
	}

	for _, c := range contexts {
		v := IsValidHealthString(c.Health)
		if v != c.Expected {
			t.Fatalf("Expected %t, but got %t", c.Expected, v)
		}
	}
}

func TestStateRunStop(t *testing.T) {
	s := NewState()

	// Begin another wait with WaitConditionRemoved. It should complete
	// within 200 milliseconds.
	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
	defer cancel()
	removalWait := s.Wait(ctx, WaitConditionRemoved)

	// Full lifecycle two times.
	for i := 1; i <= 2; i++ {
		// A wait with WaitConditionNotRunning should return
		// immediately since the state is now either "created" (on the
		// first iteration) or "exited" (on the second iteration). It
		// shouldn't take more than 50 milliseconds.
		ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
		defer cancel()
		// Expectx exit code to be i-1 since it should be the exit
		// code from the previous loop or 0 for the created state.
		if status := <-s.Wait(ctx, WaitConditionNotRunning); status.ExitCode() != i-1 {
			t.Fatalf("ExitCode %v, expected %v, err %q", status.ExitCode(), i-1, status.Err())
		}

		// A wait with WaitConditionNextExit should block until the
		// container has started and exited. It shouldn't take more
		// than 100 milliseconds.
		ctx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		initialWait := s.Wait(ctx, WaitConditionNextExit)

		// Set the state to "Running".
		s.Lock()
		s.SetRunning(i, true)
		s.Unlock()

		// Assert desired state.
		if !s.IsRunning() {
			t.Fatal("State not running")
		}
		if s.Pid != i {
			t.Fatalf("Pid %v, expected %v", s.Pid, i)
		}
		if s.ExitCode() != 0 {
			t.Fatalf("ExitCode %v, expected 0", s.ExitCode())
		}

		// Now that it's running, a wait with WaitConditionNotRunning
		// should block until we stop the container. It shouldn't take
		// more than 100 milliseconds.
		ctx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		exitWait := s.Wait(ctx, WaitConditionNotRunning)

		// Set the state to "Exited".
		s.Lock()
		s.SetStopped(&ExitStatus{ExitCode: i})
		s.Unlock()

		// Assert desired state.
		if s.IsRunning() {
			t.Fatal("State is running")
		}
		if s.ExitCode() != i {
			t.Fatalf("ExitCode %v, expected %v", s.ExitCode(), i)
		}
		if s.Pid != 0 {
			t.Fatalf("Pid %v, expected 0", s.Pid)
		}

		// Receive the initialWait result.
		if status := <-initialWait; status.ExitCode() != i {
			t.Fatalf("ExitCode %v, expected %v, err %q", status.ExitCode(), i, status.Err())
		}

		// Receive the exitWait result.
		if status := <-exitWait; status.ExitCode() != i {
			t.Fatalf("ExitCode %v, expected %v, err %q", status.ExitCode(), i, status.Err())
		}
	}

	// Set the state to dead and removed.
	s.Lock()
	s.Dead = true
	s.Unlock()
	s.SetRemoved()

	// Wait for removed status or timeout.
	if status := <-removalWait; status.ExitCode() != 2 {
		// Should have the final exit code from the loop.
		t.Fatalf("Removal wait exitCode %v, expected %v, err %q", status.ExitCode(), 2, status.Err())
	}
}

func TestStateTimeoutWait(t *testing.T) {
	s := NewState()

	s.Lock()
	s.SetRunning(0, true)
	s.Unlock()

	// Start a wait with a timeout.
	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
	defer cancel()
	waitC := s.Wait(ctx, WaitConditionNotRunning)

	// It should timeout *before* this 200ms timer does.
	select {
	case <-time.After(200 * time.Millisecond):
		t.Fatal("Stop callback doesn't fire in 200 milliseconds")
	case status := <-waitC:
		t.Log("Stop callback fired")
		// Should be a timeout error.
		if status.Err() == nil {
			t.Fatal("expected timeout error, got nil")
		}
		if status.ExitCode() != -1 {
			t.Fatalf("expected exit code %v, got %v", -1, status.ExitCode())
		}
	}

	s.Lock()
	s.SetStopped(&ExitStatus{ExitCode: 0})
	s.Unlock()

	// Start another wait with a timeout. This one should return
	// immediately.
	ctx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond)
	defer cancel()
	waitC = s.Wait(ctx, WaitConditionNotRunning)

	select {
	case <-time.After(200 * time.Millisecond):
		t.Fatal("Stop callback doesn't fire in 200 milliseconds")
	case status := <-waitC:
		t.Log("Stop callback fired")
		if status.ExitCode() != 0 {
			t.Fatalf("expected exit code %v, got %v, err %q", 0, status.ExitCode(), status.Err())
		}
	}
}

// Related issue: #39352
func TestCorrectStateWaitResultAfterRestart(t *testing.T) {
	s := NewState()

	s.Lock()
	s.SetRunning(0, true)
	s.Unlock()

	waitC := s.Wait(context.Background(), WaitConditionNotRunning)
	want := ExitStatus{ExitCode: 10, ExitedAt: time.Now()}

	s.Lock()
	s.SetRestarting(&want)
	s.Unlock()

	s.Lock()
	s.SetRunning(0, true)
	s.Unlock()

	got := <-waitC
	if got.exitCode != want.ExitCode {
		t.Fatalf("expected exit code %v, got %v", want.ExitCode, got.exitCode)
	}
}

func TestIsValidStateString(t *testing.T) {
	states := []struct {
		state    string
		expected bool
	}{
		{"paused", true},
		{"restarting", true},
		{"running", true},
		{"dead", true},
		{"start", false},
		{"created", true},
		{"exited", true},
		{"removing", true},
		{"stop", false},
	}

	for _, s := range states {
		v := IsValidStateString(s.state)
		if v != s.expected {
			t.Fatalf("Expected %t, but got %t", s.expected, v)
		}
	}
}

I don't see why this code needs to be in a separate goroutine. `waitC` is (expected by existing consumers to be) a buffered channel so there won't be any deadlocks from sending to the channel before it is received on, notwithstanding implementation bugs.