c8d: Fix `docker diff`
- fixes https://github.com/moby/moby/issues/46607

**- What I did**

Diffing a container yielded some extra changes that come from the files/directories that we mount inside the container (/etc/resolv.conf for example). To avoid that we create an intermediate snapshot that has these files, with this we can now diff the container fs with its parent and only get the differences that were made inside the container.

**- How I did it**

Added a layer for the base snapshot of a container containing the files that are created by the runtime.

**- How to verify it**

Run `TestDiff`

```console
$ make DOCKER_GRAPHDRIVER=overlayfs TEST_FILTER=TestDiff TEST_INTEGRATION_USE_SNAPSHOTTER=1 TEST_IGNORE_CGROUP_CHECK=1 test-integration
```

**- Description for the changelog**
<!--
Write a short (one line) summary that describes the changes in this
pull request for inclusion in the changelog:
-->
- containerd integration: Fixed `docker diff` containing extra differences

**- A picture of a cute animal (not mandatory but encouraged)**


daemon/containerd/image_snapshot.go
package containerd

import (
	"context"
	"fmt"

	"github.com/containerd/containerd"
	cerrdefs "github.com/containerd/containerd/errdefs"
	containerdimages "github.com/containerd/containerd/images"
	"github.com/containerd/containerd/leases"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/snapshots"
	"github.com/docker/docker/errdefs"
	"github.com/opencontainers/image-spec/identity"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

const remapSuffix = "-remap"

// PrepareSnapshot prepares a snapshot from a parent image for a container
func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *ocispec.Platform) error {
	var parentSnapshot string
	if parentImage != "" {
		img, err := i.resolveImage(ctx, parentImage)
		if err != nil {
			return err
		}

		cs := i.client.ContentStore()

		matcher := platforms.Default()
		if platform != nil {
			matcher = platforms.Only(*platform)
		}

		platformImg := containerd.NewImageWithPlatform(i.client, img, matcher)
		unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter)
		if err != nil {
			return err
		}

		if !unpacked {
			if err := platformImg.Unpack(ctx, i.snapshotter); err != nil {
				return err
			}
		}

		desc, err := containerdimages.Config(ctx, cs, img.Target, matcher)
		if err != nil {
			return err
		}

		diffIDs, err := containerdimages.RootFS(ctx, cs, desc)
		if err != nil {
			return err
		}

		parentSnapshot = identity.ChainID(diffIDs).String()
	}

	// Add a lease so that containerd doesn't garbage collect our snapshot
	ls := i.client.LeasesService()
	lease, err := ls.Create(ctx, leases.WithID(id))
	if err != nil {
		return err
	}
	if err := ls.AddResource(ctx, lease, leases.Resource{
		ID:   id,
		Type: "snapshots/" + i.StorageDriver(),
	}); err != nil {
		return err
	}

	snapshotter := i.client.SnapshotService(i.StorageDriver())

	if !i.idMapping.Empty() {
		return i.remapSnapshot(ctx, snapshotter, id, parentSnapshot, lease)
	}

	_, err = snapshotter.Prepare(ctx, id, parentSnapshot)
	return err
}

// calculateSnapshotParentUsage returns the usage of all ancestors of the
// provided snapshot. It doesn't include the size of the snapshot itself.
func calculateSnapshotParentUsage(ctx context.Context, snapshotter snapshots.Snapshotter, snapshotID string) (snapshots.Usage, error) {
	info, err := snapshotter.Stat(ctx, snapshotID)
	if err != nil {
		if cerrdefs.IsNotFound(err) {
			return snapshots.Usage{}, errdefs.NotFound(err)
		}
		return snapshots.Usage{}, errdefs.System(errors.Wrapf(err, "snapshotter.Stat failed for %s", snapshotID))
	}
	if info.Parent == "" {
		return snapshots.Usage{}, errdefs.NotFound(fmt.Errorf("snapshot %s has no parent", snapshotID))
	}

	return calculateSnapshotTotalUsage(ctx, snapshotter, info.Parent)
}

// calculateSnapshotTotalUsage returns the total usage of that snapshot
// including all of its ancestors.
func calculateSnapshotTotalUsage(ctx context.Context, snapshotter snapshots.Snapshotter, snapshotID string) (snapshots.Usage, error) {
	var total snapshots.Usage
	next := snapshotID

	for next != "" {
		usage, err := snapshotter.Usage(ctx, next)
		if err != nil {
			if cerrdefs.IsNotFound(err) {
				return total, errdefs.NotFound(errors.Wrapf(err, "non-existing ancestor of %s", snapshotID))
			}
			return total, errdefs.System(errors.Wrapf(err, "snapshotter.Usage failed for %s", next))
		}
		total.Size += usage.Size
		total.Inodes += usage.Inodes

		info, err := snapshotter.Stat(ctx, next)
		if err != nil {
			return total, errdefs.System(errors.Wrapf(err, "snapshotter.Stat failed for %s", next))
		}
		next = info.Parent
	}
	return total, nil
}

package containerd

import (
	"context"
	"fmt"

	"github.com/containerd/containerd"
	cerrdefs "github.com/containerd/containerd/errdefs"
	containerdimages "github.com/containerd/containerd/images"
	"github.com/containerd/containerd/leases"
	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/snapshots"
	"github.com/docker/docker/errdefs"
	"github.com/opencontainers/image-spec/identity"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

const remapSuffix = "-remap"

// PrepareSnapshot prepares a snapshot from a parent image for a container
func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *ocispec.Platform, setupInit func(string) error) error {
	var parentSnapshot string
	if parentImage != "" {
		img, err := i.resolveImage(ctx, parentImage)
		if err != nil {
			return err
		}

		cs := i.client.ContentStore()

		matcher := platforms.Default()
		if platform != nil {
			matcher = platforms.Only(*platform)
		}

		platformImg := containerd.NewImageWithPlatform(i.client, img, matcher)
		unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter)
		if err != nil {
			return err
		}

		if !unpacked {
			if err := platformImg.Unpack(ctx, i.snapshotter); err != nil {
				return err
			}
		}

		desc, err := containerdimages.Config(ctx, cs, img.Target, matcher)
		if err != nil {
			return err
		}

		diffIDs, err := containerdimages.RootFS(ctx, cs, desc)
		if err != nil {
			return err
		}

		parentSnapshot = identity.ChainID(diffIDs).String()
	}

	ls := i.client.LeasesService()
	lease, err := ls.Create(ctx, leases.WithID(id))
	if err != nil {
		return err
	}
	ctx = leases.WithLease(ctx, lease.ID)

	snapshotter := i.client.SnapshotService(i.StorageDriver())

	if err := i.prepareInitLayer(ctx, id, parentSnapshot, setupInit); err != nil {
		return err
	}

	if !i.idMapping.Empty() {
		return i.remapSnapshot(ctx, snapshotter, id, id+"-init")
	}

	_, err = snapshotter.Prepare(ctx, id, id+"-init")
	return err
}

func (i *ImageService) prepareInitLayer(ctx context.Context, id string, parent string, setupInit func(string) error) error {
	snapshotter := i.client.SnapshotService(i.StorageDriver())

	mounts, err := snapshotter.Prepare(ctx, id+"-init-key", parent)
	if err != nil {
		return err
	}

	if err := mount.WithTempMount(ctx, mounts, func(root string) error {
		return setupInit(root)
	}); err != nil {
		return err
	}

	return snapshotter.Commit(ctx, id+"-init", id+"-init-key")
}

// calculateSnapshotParentUsage returns the usage of all ancestors of the
// provided snapshot. It doesn't include the size of the snapshot itself.
func calculateSnapshotParentUsage(ctx context.Context, snapshotter snapshots.Snapshotter, snapshotID string) (snapshots.Usage, error) {
	info, err := snapshotter.Stat(ctx, snapshotID)
	if err != nil {
		if cerrdefs.IsNotFound(err) {
			return snapshots.Usage{}, errdefs.NotFound(err)
		}
		return snapshots.Usage{}, errdefs.System(errors.Wrapf(err, "snapshotter.Stat failed for %s", snapshotID))
	}
	if info.Parent == "" {
		return snapshots.Usage{}, errdefs.NotFound(fmt.Errorf("snapshot %s has no parent", snapshotID))
	}

	return calculateSnapshotTotalUsage(ctx, snapshotter, info.Parent)
}

// calculateSnapshotTotalUsage returns the total usage of that snapshot
// including all of its ancestors.
func calculateSnapshotTotalUsage(ctx context.Context, snapshotter snapshots.Snapshotter, snapshotID string) (snapshots.Usage, error) {
	var total snapshots.Usage
	next := snapshotID

	for next != "" {
		usage, err := snapshotter.Usage(ctx, next)
		if err != nil {
			if cerrdefs.IsNotFound(err) {
				return total, errdefs.NotFound(errors.Wrapf(err, "non-existing ancestor of %s", snapshotID))
			}
			return total, errdefs.System(errors.Wrapf(err, "snapshotter.Usage failed for %s", next))
		}
		total.Size += usage.Size
		total.Inodes += usage.Inodes

		info, err := snapshotter.Stat(ctx, next)
		if err != nil {
			return total, errdefs.System(errors.Wrapf(err, "snapshotter.Stat failed for %s", next))
		}
		next = info.Parent
	}
	return total, nil
}

Are those parts that are missing the "default mounts"? Wondering if in that case we could take that list and exclude them 🤔