c8d/prune: Remove gc.ref labels from configs of deleted images 
- Follow up to: https://github.com/moby/moby/pull/45270

Temporary workaround for: https://github.com/moby/buildkit/issues/3797

**- What I did**
Fixed snapshots of unpacked images not being deleted

**- How I did it**
Manually remove gc.ref labels from configs which are not referenced by any image.

**- How to verify it**


<details>

<summary>docker rmi</summary>

```bash
$ echo 'FROM ubuntu' | docker build -qt asdf -
docker buildx psha256:2c810c8982bab60cb661704546104727f97a31f3ad8c451e1d2e212216ceff12

$ docker buildx prune -a
WARNING! This will remove all build cache. Are you sure you want to continue? [y/N] y
ID                                              RECLAIMABLE     SIZE            LAST ACCESSED
2s5mcpdcorpvthkdfoniedjot*                      true            8.192kB         1 second ago
i82yyjvqh889nd1ua3u9iwdq8*                      true    4.096kB         1 second ago
jgghwgr5rzotfa5rzsuwa2r1o                       true    27.35MB         1 second ago
Total:  27.36MB

$ ctr snapshot ls
KEY                                                                     PARENT KIND
sha256:874b048c963ab55b06939c39d59303fb975d323822a4ea48a02ac8dc635ea371        Committed

$ docker rmi asdf
Untagged: asdf

$ ctr snapshot ls
KEY PARENT KIND
```

</details>




<details>

<summary>docker image prune</summary>

```bash
$ echo 'FROM ubuntu' | docker build -qt asdf -
sha256:8b15904a219973abd67d9ed69902053e8e07cd81ee10a88d5d93f1722cabaa13

$ docker buildx prune -a
WARNING! This will remove all build cache. Are you sure you want to continue? [y/N] y
ID                                              RECLAIMABLE     SIZE            LAST ACCESSED
y92vqiqt18vuscz15q43ee0f9*                      true            4.096kB         3 seconds ago
7e2ymvv65rtif3vabl1dt45j3*                      true    8.192kB         3 seconds ago
tymibv3erhje6pm3ucm0czln5                       true    27.35MB         3 seconds ago
Total:  27.36MB

$ ctr snapshot ls
KEY                                                                     PARENT KIND
sha256:874b048c963ab55b06939c39d59303fb975d323822a4ea48a02ac8dc635ea371        Committed

$ docker image prune -a
WARNING! This will remove all images without at least one container associated to them.
Are you sure you want to continue? [y/N] y
Deleted Images:
untagged: docker.io/library/asdf:latest

Total reclaimed space: 0B

$ ctr snapshot ls
KEY PARENT KIND
```
</details>




**- Description for the changelog**
<!--
Write a short (one line) summary that describes the changes in this
pull request for inclusion in the changelog:
-->


**- A picture of a cute animal (not mandatory but encouraged)**


daemon/containerd/image_delete.go
package containerd

import (
	"context"

	"github.com/containerd/containerd/images"
	"github.com/docker/distribution/reference"
	"github.com/docker/docker/api/types"
)

// ImageDelete deletes the image referenced by the given imageRef from this
// daemon. The given imageRef can be an image ID, ID prefix, or a repository
// reference (with an optional tag or digest, defaulting to the tag name
// "latest"). There is differing behavior depending on whether the given
// imageRef is a repository reference or not.
//
// If the given imageRef is a repository reference then that repository
// reference will be removed. However, if there exists any containers which
// were created using the same image reference then the repository reference
// cannot be removed unless either there are other repository references to the
// same image or force is true. Following removal of the repository reference,
// the referenced image itself will attempt to be deleted as described below
// but quietly, meaning any image delete conflicts will cause the image to not
// be deleted and the conflict will not be reported.
//
// There may be conflicts preventing deletion of an image and these conflicts
// are divided into two categories grouped by their severity:
//
// Hard Conflict:
//   - a pull or build using the image.
//   - any descendant image.
//   - any running container using the image.
//
// Soft Conflict:
//   - any stopped container using the image.
//   - any repository tag or digest references to the image.
//
// The image cannot be removed if there are any hard conflicts and can be
// removed if there are soft conflicts only if force is true.
//
// If prune is true, ancestor images will each attempt to be deleted quietly,
// meaning any delete conflicts will cause the image to not be deleted and the
// conflict will not be reported.
//
// TODO(thaJeztah): implement ImageDelete "force" options; see https://github.com/moby/moby/issues/43850
// TODO(thaJeztah): implement ImageDelete "prune" options; see https://github.com/moby/moby/issues/43849
// TODO(thaJeztah): image delete should send prometheus counters; see https://github.com/moby/moby/issues/45268
func (i *ImageService) ImageDelete(ctx context.Context, imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) {
	parsedRef, err := reference.ParseNormalizedNamed(imageRef)
	if err != nil {
		return nil, err
	}

	img, err := i.resolveImage(ctx, imageRef)
	if err != nil {
		return nil, err
	}

	err = i.client.ImageService().Delete(ctx, img.Name, images.SynchronousDelete())
	if err != nil {
		return nil, err
	}

	imgID := string(img.Target.Digest)
	i.LogImageEvent(imgID, imgID, "untag")
	i.LogImageEvent(imgID, imgID, "delete")

	return []types.ImageDeleteResponseItem{{Untagged: reference.FamiliarString(parsedRef)}}, nil
}

package containerd

import (
	"context"

	"github.com/containerd/containerd/images"
	"github.com/docker/distribution/reference"
	"github.com/docker/docker/api/types"
	"github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/sirupsen/logrus"
)

// ImageDelete deletes the image referenced by the given imageRef from this
// daemon. The given imageRef can be an image ID, ID prefix, or a repository
// reference (with an optional tag or digest, defaulting to the tag name
// "latest"). There is differing behavior depending on whether the given
// imageRef is a repository reference or not.
//
// If the given imageRef is a repository reference then that repository
// reference will be removed. However, if there exists any containers which
// were created using the same image reference then the repository reference
// cannot be removed unless either there are other repository references to the
// same image or force is true. Following removal of the repository reference,
// the referenced image itself will attempt to be deleted as described below
// but quietly, meaning any image delete conflicts will cause the image to not
// be deleted and the conflict will not be reported.
//
// There may be conflicts preventing deletion of an image and these conflicts
// are divided into two categories grouped by their severity:
//
// Hard Conflict:
//   - a pull or build using the image.
//   - any descendant image.
//   - any running container using the image.
//
// Soft Conflict:
//   - any stopped container using the image.
//   - any repository tag or digest references to the image.
//
// The image cannot be removed if there are any hard conflicts and can be
// removed if there are soft conflicts only if force is true.
//
// If prune is true, ancestor images will each attempt to be deleted quietly,
// meaning any delete conflicts will cause the image to not be deleted and the
// conflict will not be reported.
//
// TODO(thaJeztah): implement ImageDelete "force" options; see https://github.com/moby/moby/issues/43850
// TODO(thaJeztah): implement ImageDelete "prune" options; see https://github.com/moby/moby/issues/43849
// TODO(thaJeztah): image delete should send prometheus counters; see https://github.com/moby/moby/issues/45268
func (i *ImageService) ImageDelete(ctx context.Context, imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) {
	parsedRef, err := reference.ParseNormalizedNamed(imageRef)
	if err != nil {
		return nil, err
	}

	img, err := i.resolveImage(ctx, imageRef)
	if err != nil {
		return nil, err
	}

	possiblyDeletedConfigs := map[digest.Digest]struct{}{}
	if err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, d ocispec.Descriptor) {
		if images.IsConfigType(d.MediaType) {
			possiblyDeletedConfigs[d.Digest] = struct{}{}
		}
	}); err != nil {
		return nil, err
	}

	err = i.client.ImageService().Delete(ctx, img.Name, images.SynchronousDelete())
	if err != nil {
		return nil, err
	}

	// Workaround for: https://github.com/moby/buildkit/issues/3797
	if err := i.unleaseSnapshotsFromDeletedConfigs(context.Background(), possiblyDeletedConfigs); err != nil {
		logrus.WithError(err).Warn("failed to unlease snapshots")
	}

	imgID := string(img.Target.Digest)
	i.LogImageEvent(imgID, imgID, "untag")
	i.LogImageEvent(imgID, imgID, "delete")

	return []types.ImageDeleteResponseItem{{Untagged: reference.FamiliarString(parsedRef)}}, nil
}

What are the consequences of failing to unlease the snapshots? I get the impression that a rootfs snapshot could leak. And since the image has already been deleted at this point, the user would not be able to retry. I presume that the only recourse would be to use a low-level tool such as `ctr` to poke the containerd API directly?

Would it be possible to unlease the snapshots before the image is deleted such that the image is only deleted if the unleasing succeeds?