c8d: Handle userns properly
**- What I did**

Remapped the GID/UID when a snapshot is created if the daemon is started with `--userns-remap`. This implementation differs from the graph driver one slightly. With graph drivers the files/directories are chown'ed when the image is downloaded and the contents extracted. Basically, with graph drivers we remap on pull and with containerd we remap on run.

Note: with this change builds still fail, my guess is because buildkit still mounts the roots in normal mode, I'll continue looking but I think we can merge this one, I am pretty sure some changes in buildkit are needed:

```console
$ docker build -t userns . 
[+] Building 1.7s (6/6) FINISHED                                                                                                                               docker:local
 => [internal] load build definition from dd                                                                                                                           0.0s
 => => transferring dockerfile: 71B                                                                                                                                    0.0s
 => [internal] load .dockerignore                                                                                                                                      0.0s
 => => transferring context: 2B                                                                                                                                        0.0s
 => [internal] load metadata for docker.io/library/debian:bullseye-slim                                                                                                1.3s
 => [auth] library/debian:pull token for registry-1.docker.io                                                                                                          0.0s
 => CACHED [1/2] FROM docker.io/library/debian:bullseye-slim@sha256:61386e11b5256efa33823cbfafd668dd651dbce810b24a8fb7b2e32fa7f65a85                                   0.0s
 => => resolve docker.io/library/debian:bullseye-slim@sha256:61386e11b5256efa33823cbfafd668dd651dbce810b24a8fb7b2e32fa7f65a85                                          0.0s
 => ERROR [2/2] RUN touch testa                                                                                                                                        0.3s
------
 > [2/2] RUN touch testa:
0.287 runc run failed: unable to start container process: error during container init: error mounting "/tmp/root/165536.165536/buildkit/executor/hosts.iv21cdui6qvdwqel9cdo93spo" to rootfs at "/etc/hosts": open /tmp/root/165536.165536/buildkit/executor/7yecjv2i25qdu44khgx41b450/rootfs/etc/hosts: permission denied
------
dd:2
--------------------
   1 |     FROM debian:bullseye-slim
   2 | >>> RUN touch testa
   3 |     
--------------------
ERROR: failed to solve: process "/bin/sh -c touch testa" did not complete successfully: exit code: 1
```

**- How I did it**

**- How to verify it**

`TestDaemonUserNamespaceRootSetting` should not fail any more
 
```console
$ make DOCKER_GRAPHDRIVER=overlayfs TEST_FILTER=TestDaemonUserNamespaceRootSetting TEST_INTEGRATION_USE_SNAPSHOTTER=1 test-integration
```

**- Description for the changelog**
<!--
Write a short (one line) summary that describes the changes in this
pull request for inclusion in the changelog:
-->


**- A picture of a cute animal (not mandatory but encouraged)**

daemon/snapshotter/mount.go
package snapshotter

import (
	"context"
	"os"
	"path/filepath"

	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/mount"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/idtools"
	"github.com/moby/locker"
)

const mountsDir = "rootfs"

// List of known filesystems that can't be re-mounted or have shared layers
var refCountedFileSystems = []string{"fuse-overlayfs", "overlayfs", "stargz", "zfs"}

// Mounter handles mounting/unmounting things coming in from a snapshotter
// with optional reference counting if needed by the filesystem
type Mounter interface {
	// Mount mounts the rootfs for a container and returns the mount point
	Mount(mounts []mount.Mount, containerID string) (string, error)
	// Unmount unmounts the container rootfs
	Unmount(target string) error
}

// inSlice tests whether a string is contained in a slice of strings or not.
// Comparison is case sensitive
func inSlice(slice []string, s string) bool {
	for _, ss := range slice {
		if s == ss {
			return true
		}
	}
	return false
}

// NewMounter creates a new mounter for the provided snapshotter
func NewMounter(home string, snapshotter string, idMap idtools.IdentityMapping) Mounter {
	if inSlice(refCountedFileSystems, snapshotter) {
		return &refCountMounter{
			home:        home,
			snapshotter: snapshotter,
			rc:          graphdriver.NewRefCounter(checker()),
			locker:      locker.New(),
			idMap:       idMap,
		}
	}

	return mounter{
		home:        home,
		snapshotter: snapshotter,
		idMap:       idMap,
	}
}

type refCountMounter struct {
	home        string
	snapshotter string
	rc          *graphdriver.RefCounter
	locker      *locker.Locker
	idMap       idtools.IdentityMapping
}

func (m *refCountMounter) Mount(mounts []mount.Mount, containerID string) (target string, retErr error) {
	target = filepath.Join(m.home, mountsDir, m.snapshotter, containerID)

	_, err := os.Stat(target)
	if err != nil && !os.IsNotExist(err) {
		return "", err
	}

	if count := m.rc.Increment(target); count > 1 {
		return target, nil
	}

	m.locker.Lock(target)
	defer m.locker.Unlock(target)

	defer func() {
		if retErr != nil {
			if c := m.rc.Decrement(target); c <= 0 {
				if mntErr := unmount(target); mntErr != nil {
					log.G(context.TODO()).Errorf("error unmounting %s: %v", target, mntErr)
				}
				if rmErr := os.Remove(target); rmErr != nil && !os.IsNotExist(rmErr) {
					log.G(context.TODO()).Debugf("Failed to remove %s: %v: %v", target, rmErr, err)
				}
			}
		}
	}()

	root := m.idMap.RootPair()
	if err := idtools.MkdirAllAndChown(target, 0o700, root); err != nil {
		return "", err
	}

	return target, mount.All(mounts, target)
}

func (m *refCountMounter) Unmount(target string) error {
	if count := m.rc.Decrement(target); count > 0 {
		return nil
	}

	m.locker.Lock(target)
	defer m.locker.Unlock(target)

	if err := unmount(target); err != nil {
		log.G(context.TODO()).Debugf("Failed to unmount %s: %v", target, err)
	}

	if err := os.Remove(target); err != nil {
		log.G(context.TODO()).WithError(err).WithField("dir", target).Error("failed to remove mount temp dir")
	}

	return nil
}

type mounter struct {
	home        string
	snapshotter string
	idMap       idtools.IdentityMapping
}

func (m mounter) Mount(mounts []mount.Mount, containerID string) (string, error) {
	target := filepath.Join(m.home, mountsDir, m.snapshotter, containerID)

	root := m.idMap.RootPair()
	if err := idtools.MkdirAndChown(target, 0o700, root); err != nil {
		return "", err
	}

	return target, mount.All(mounts, target)
}

func (m mounter) Unmount(target string) error {
	return unmount(target)
}

package snapshotter

import (
	"context"
	"os"
	"path/filepath"

	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/mount"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/idtools"
	"github.com/moby/locker"
)

const mountsDir = "rootfs"

// List of known filesystems that can't be re-mounted or have shared layers
var refCountedFileSystems = []string{"fuse-overlayfs", "overlayfs", "stargz", "zfs"}

// Mounter handles mounting/unmounting things coming in from a snapshotter
// with optional reference counting if needed by the filesystem
type Mounter interface {
	// Mount mounts the rootfs for a container and returns the mount point
	Mount(mounts []mount.Mount, containerID string) (string, error)
	// Unmount unmounts the container rootfs
	Unmount(target string) error
}

// inSlice tests whether a string is contained in a slice of strings or not.
// Comparison is case sensitive
func inSlice(slice []string, s string) bool {
	for _, ss := range slice {
		if s == ss {
			return true
		}
	}
	return false
}

// NewMounter creates a new mounter for the provided snapshotter
func NewMounter(home string, snapshotter string, idMap idtools.IdentityMapping) Mounter {
	if inSlice(refCountedFileSystems, snapshotter) {
		return &refCountMounter{
			home:        home,
			snapshotter: snapshotter,
			rc:          graphdriver.NewRefCounter(checker()),
			locker:      locker.New(),
			idMap:       idMap,
		}
	}

	return mounter{
		home:        home,
		snapshotter: snapshotter,
		idMap:       idMap,
	}
}

type refCountMounter struct {
	home        string
	snapshotter string
	rc          *graphdriver.RefCounter
	locker      *locker.Locker
	idMap       idtools.IdentityMapping
}

func (m *refCountMounter) Mount(mounts []mount.Mount, containerID string) (target string, retErr error) {
	target = filepath.Join(m.home, mountsDir, m.snapshotter, containerID)

	_, err := os.Stat(target)
	if err != nil && !os.IsNotExist(err) {
		return "", err
	}

	if count := m.rc.Increment(target); count > 1 {
		return target, nil
	}

	m.locker.Lock(target)
	defer m.locker.Unlock(target)

	defer func() {
		if retErr != nil {
			if c := m.rc.Decrement(target); c <= 0 {
				if mntErr := unmount(target); mntErr != nil {
					log.G(context.TODO()).Errorf("error unmounting %s: %v", target, mntErr)
				}
				if rmErr := os.Remove(target); rmErr != nil && !os.IsNotExist(rmErr) {
					log.G(context.TODO()).Debugf("Failed to remove %s: %v: %v", target, rmErr, err)
				}
			}
		}
	}()

	root := m.idMap.RootPair()
	if err := idtools.MkdirAllAndChown(filepath.Dir(target), 0o710, idtools.Identity{
		UID: idtools.CurrentIdentity().UID,
		GID: root.GID,
	}); err != nil {
		return "", err
	}
	if err := idtools.MkdirAllAndChown(target, 0o710, root); err != nil {
		return "", err
	}

	return target, mount.All(mounts, target)
}

func (m *refCountMounter) Unmount(target string) error {
	if count := m.rc.Decrement(target); count > 0 {
		return nil
	}

	m.locker.Lock(target)
	defer m.locker.Unlock(target)

	if err := unmount(target); err != nil {
		log.G(context.TODO()).Debugf("Failed to unmount %s: %v", target, err)
	}

	if err := os.Remove(target); err != nil {
		log.G(context.TODO()).WithError(err).WithField("dir", target).Error("failed to remove mount temp dir")
	}

	return nil
}

type mounter struct {
	home        string
	snapshotter string
	idMap       idtools.IdentityMapping
}

func (m mounter) Mount(mounts []mount.Mount, containerID string) (string, error) {
	target := filepath.Join(m.home, mountsDir, m.snapshotter, containerID)

	root := m.idMap.RootPair()
	if err := idtools.MkdirAndChown(target, 0o700, root); err != nil {
		return "", err
	}

	return target, mount.All(mounts, target)
}

func (m mounter) Unmount(target string) error {
	return unmount(target)
}

🙈 looks like your touch-up ended in the second commit instead of the first 🙈 