c8d: Fix image commit with userns mapping (carry)
Alternative approach to #46938 as mentioned [here](https://github.com/moby/moby/pull/46938#issuecomment-1865249630).
daemon/containerd/image_snapshot_unix.go
//go:build !windows

package containerd

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/snapshots"
	"github.com/containerd/log"
	"github.com/docker/docker/pkg/idtools"
)

func (i *ImageService) remapSnapshot(ctx context.Context, snapshotter snapshots.Snapshotter, id string, parentSnapshot string) error {
	rootPair := i.idMapping.RootPair()
	usernsID := fmt.Sprintf("%s-%d-%d", parentSnapshot, rootPair.UID, rootPair.GID)
	remappedID := usernsID + remapSuffix

	// If the remapped snapshot already exist we only need to prepare the new snapshot
	if _, err := snapshotter.Stat(ctx, usernsID); err == nil {
		_, err = snapshotter.Prepare(ctx, id, usernsID)
		return err
	}

	mounts, err := snapshotter.Prepare(ctx, remappedID, parentSnapshot)
	if err != nil {
		return err
	}

	if err := i.remapRootFS(ctx, mounts); err != nil {
		if rmErr := snapshotter.Remove(ctx, usernsID); rmErr != nil {
			log.G(ctx).WithError(rmErr).Warn("failed to remove snapshot after remap error")
		}
		return err
	}

	if err := snapshotter.Commit(ctx, usernsID, remappedID); err != nil {
		return err
	}

	_, err = snapshotter.Prepare(ctx, id, usernsID)
	return err
}

func (i *ImageService) remapRootFS(ctx context.Context, mounts []mount.Mount) error {
	return mount.WithTempMount(ctx, mounts, func(root string) error {
		return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			stat := info.Sys().(*syscall.Stat_t)
			if stat == nil {
				return fmt.Errorf("cannot get underlying data for %s", path)
			}

			ids, err := i.idMapping.ToHost(idtools.Identity{UID: int(stat.Uid), GID: int(stat.Gid)})
			if err != nil {
				return err
			}

			return os.Lchown(path, ids.UID, ids.GID)
		})
	})
}

func (i *ImageService) unremapRootFS(ctx context.Context, mounts []mount.Mount) error {
	return mount.WithTempMount(ctx, mounts, func(root string) error {
		return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			stat := info.Sys().(*syscall.Stat_t)
			if stat == nil {
				return fmt.Errorf("cannot get underlying data for %s", path)
			}

			uid, gid, err := i.idMapping.ToContainer(idtools.Identity{UID: int(stat.Uid), GID: int(stat.Gid)})
			if err != nil {
				return err
			}

			return os.Lchown(path, uid, gid)
		})
	})
}

//go:build !windows

package containerd

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/snapshots"
	"github.com/containerd/continuity/fs"
	"github.com/containerd/continuity/sysx"
	"github.com/docker/docker/pkg/idtools"
)

const (
	// Values based on linux/include/uapi/linux/capability.h
	xattrCapsSz2    = 20
	versionOffset   = 3
	vfsCapRevision2 = 2
	vfsCapRevision3 = 3
	remapSuffix     = "-remap"
)

func (i *ImageService) remapSnapshot(ctx context.Context, snapshotter snapshots.Snapshotter, id string, parentSnapshot string) error {
	_, err := snapshotter.Prepare(ctx, id, parentSnapshot)
	if err != nil {
		return err
	}
	mounts, err := snapshotter.Mounts(ctx, id)
	if err != nil {
		return err
	}

	if err := i.remapRootFS(ctx, mounts); err != nil {
		return err
	}

	return err
}

func (i *ImageService) remapRootFS(ctx context.Context, mounts []mount.Mount) error {
	return mount.WithTempMount(ctx, mounts, func(root string) error {
		return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			stat := info.Sys().(*syscall.Stat_t)
			if stat == nil {
				return fmt.Errorf("cannot get underlying data for %s", path)
			}

			ids, err := i.idMapping.ToHost(idtools.Identity{UID: int(stat.Uid), GID: int(stat.Gid)})
			if err != nil {
				return err
			}

			return chownWithCaps(path, ids.UID, ids.GID)
		})
	})
}

func (i *ImageService) copyAndUnremapRootFS(ctx context.Context, dst, src []mount.Mount) error {
	return mount.WithTempMount(ctx, src, func(source string) error {
		return mount.WithTempMount(ctx, dst, func(root string) error {
			// TODO: Update CopyDir to support remap directly
			if err := fs.CopyDir(root, source); err != nil {
				return fmt.Errorf("failed to copy: %w", err)
			}

			return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}

				stat := info.Sys().(*syscall.Stat_t)
				if stat == nil {
					return fmt.Errorf("cannot get underlying data for %s", path)
				}

				uid, gid, err := i.idMapping.ToContainer(idtools.Identity{UID: int(stat.Uid), GID: int(stat.Gid)})
				if err != nil {
					return err
				}

				return chownWithCaps(path, uid, gid)
			})
		})
	})
}

func (i *ImageService) unremapRootFS(ctx context.Context, mounts []mount.Mount) error {
	return mount.WithTempMount(ctx, mounts, func(root string) error {
		return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			stat := info.Sys().(*syscall.Stat_t)
			if stat == nil {
				return fmt.Errorf("cannot get underlying data for %s", path)
			}

			uid, gid, err := i.idMapping.ToContainer(idtools.Identity{UID: int(stat.Uid), GID: int(stat.Gid)})
			if err != nil {
				return err
			}

			return chownWithCaps(path, uid, gid)
		})
	})
}

// chownWithCaps will chown path and preserve the extended attributes.
// chowning a file will remove the capabilities, so we need to first get all of
// them, chown the file, and then set the extended attributes
func chownWithCaps(path string, uid int, gid int) error {
	xattrKeys, err := sysx.LListxattr(path)
	if err != nil {
		return err
	}

	xattrs := make(map[string][]byte, len(xattrKeys))

	for _, xattr := range xattrKeys {
		data, err := sysx.LGetxattr(path, xattr)
		if err != nil {
			return err
		}
		xattrs[xattr] = data
	}

	if err := os.Lchown(path, uid, gid); err != nil {
		return err
	}

	for xattrKey, xattrValue := range xattrs {
		length := len(xattrValue)
		// make sure the capabilities are version 2,
		// capabilities version 3 also store the root uid of the namespace,
		// we don't want this when we are in userns-remap mode
		// see: https://github.com/moby/moby/pull/41724
		if xattrKey == "security.capability" && xattrValue[versionOffset] == vfsCapRevision3 {
			xattrValue[versionOffset] = vfsCapRevision2
			length = xattrCapsSz2
		}
		if err := sysx.LSetxattr(path, xattrKey, xattrValue[:length], 0); err != nil {
			return err
		}
	}

	return nil
}

Can be in a follow-up, but considering to add some details about why these attributes are stripped in the first place (I was initially scratching my head on that), and why we have to go against that. Perhaps it's considered "common knowledge" but I wasn't aware (🙈) Djordje posted a link to https://lwn.net/Articles/244747/ on Slack, which made that clear to me.