<commit_before>package storage

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/containers/storage/pkg/ioutils"
	"github.com/containers/storage/pkg/stringid"
	"github.com/containers/storage/pkg/stringutils"
	"github.com/containers/storage/pkg/truncindex"
	digest "github.com/opencontainers/go-digest"
)

const (
	// ImageDigestManifestBigDataNamePrefix is a prefix of big data item
	// names which we consider to be manifests, used for computing a
	// "digest" value for the image as a whole, by which we can locate the
	// image later.
	ImageDigestManifestBigDataNamePrefix = "manifest"
	// ImageDigestBigDataKey is provided for compatibility with older
	// versions of the image library.  It will be removed in the future.
	ImageDigestBigDataKey = "manifest"
)

// An Image is a reference to a layer and an associated metadata string.
type Image struct {
	// ID is either one which was specified at create-time, or a random
	// value which was generated by the library.
	ID string `json:"id"`

	// Digest is a digest value that we can use to locate the image, if one
	// was specified at creation-time.
	Digest digest.Digest `json:"digest,omitempty"`

	// Digests is a list of digest values of the image's manifests, and
	// possibly a manually-specified value, that we can use to locate the
	// image.  If Digest is set, its value is also in this list.
	Digests []digest.Digest `json:"-"`

	// Names is an optional set of user-defined convenience values.  The
	// image can be referred to by its ID or any of its names.  Names are
	// unique among images, and are often the text representation of tagged
	// or canonical references.
	Names []string `json:"names,omitempty"`

	// NamesHistory is an optional set of Names the image had in the past. The
	// contained names are free from any duplicates, whereas the newest entry
	// is the first one.
	NamesHistory []string `json:"names-history,omitempty"`

	// TopLayer is the ID of the topmost layer of the image itself, if the
	// image contains one or more layers.  Multiple images can refer to the
	// same top layer.
	TopLayer string `json:"layer,omitempty"`

	// MappedTopLayers are the IDs of alternate versions of the top layer
	// which have the same contents and parent, and which differ from
	// TopLayer only in which ID mappings they use.  When the image is
	// to be removed, they should be removed before the TopLayer, as the
	// graph driver may depend on that.
	MappedTopLayers []string `json:"mapped-layers,omitempty"`

	// Metadata is data we keep for the convenience of the caller.  It is not
	// expected to be large, since it is kept in memory.
	Metadata string `json:"metadata,omitempty"`

	// BigDataNames is a list of names of data items that we keep for the
	// convenience of the caller.  They can be large, and are only in
	// memory when being read from or written to disk.
	BigDataNames []string `json:"big-data-names,omitempty"`

	// BigDataSizes maps the names in BigDataNames to the sizes of the data
	// that has been stored, if they're known.
	BigDataSizes map[string]int64 `json:"big-data-sizes,omitempty"`

	// BigDataDigests maps the names in BigDataNames to the digests of the
	// data that has been stored, if they're known.
	BigDataDigests map[string]digest.Digest `json:"big-data-digests,omitempty"`

	// Created is the datestamp for when this image was created.  Older
	// versions of the library did not track this information, so callers
	// will likely want to use the IsZero() method to verify that a value
	// is set before using it.
	Created time.Time `json:"created,omitempty"`

	// ReadOnly is true if this image resides in a read-only layer store.
	ReadOnly bool `json:"-"`

	Flags map[string]interface{} `json:"flags,omitempty"`
}

// roImageStore provides bookkeeping for information about Images.
type roImageStore interface {
	roMetadataStore
	roBigDataStore

	// startReading makes sure the store is fresh, and locks it for reading.
	// If this succeeds, the caller MUST call stopReading().
	startReading() error

	// stopReading releases locks obtained by startReading.
	stopReading()

	// Exists checks if there is an image with the given ID or name.
	Exists(id string) bool

	// Get retrieves information about an image given an ID or name.
	Get(id string) (*Image, error)

	// Images returns a slice enumerating the known images.
	Images() ([]Image, error)

	// ByDigest returns a slice enumerating the images which have either an
	// explicitly-set digest, or a big data item with a name that starts
	// with ImageDigestManifestBigDataNamePrefix, which matches the
	// specified digest.
	ByDigest(d digest.Digest) ([]*Image, error)
}

// rwImageStore provides bookkeeping for information about Images.
type rwImageStore interface {
	roImageStore
	rwMetadataStore
	rwImageBigDataStore
	flaggableStore

	// startWriting makes sure the store is fresh, and locks it for writing.
	// If this succeeds, the caller MUST call stopWriting().
	startWriting() error

	// stopWriting releases locks obtained by startWriting.
	stopWriting()

	// Create creates an image that has a specified ID (or a random one) and
	// optional names, using the specified layer as its topmost (hopefully
	// read-only) layer.  That layer can be referenced by multiple images.
	Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (*Image, error)

	// updateNames modifies names associated with an image based on (op, names).
	// The values are expected to be valid normalized
	// named image references.
	updateNames(id string, names []string, op updateNameOperation) error

	// Delete removes the record of the image.
	Delete(id string) error

	// Wipe removes records of all images.
	Wipe() error
}

type imageStore struct {
	lockfile Locker // lockfile.IsReadWrite can be used to distinguish between read-write and read-only image stores.
	dir      string
	images   []*Image
	idindex  *truncindex.TruncIndex
	byid     map[string]*Image
	byname   map[string]*Image
	bydigest map[digest.Digest][]*Image
	loadMut  sync.Mutex
}

func copyImage(i *Image) *Image {
	return &Image{
		ID:              i.ID,
		Digest:          i.Digest,
		Digests:         copyDigestSlice(i.Digests),
		Names:           copyStringSlice(i.Names),
		NamesHistory:    copyStringSlice(i.NamesHistory),
		TopLayer:        i.TopLayer,
		MappedTopLayers: copyStringSlice(i.MappedTopLayers),
		Metadata:        i.Metadata,
		BigDataNames:    copyStringSlice(i.BigDataNames),
		BigDataSizes:    copyStringInt64Map(i.BigDataSizes),
		BigDataDigests:  copyStringDigestMap(i.BigDataDigests),
		Created:         i.Created,
		ReadOnly:        i.ReadOnly,
		Flags:           copyStringInterfaceMap(i.Flags),
	}
}

func copyImageSlice(slice []*Image) []*Image {
	if len(slice) > 0 {
		cp := make([]*Image, len(slice))
		for i := range slice {
			cp[i] = copyImage(slice[i])
		}
		return cp
	}
	return nil
}

// startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
//
// This is an internal implementation detail of imageStore construction, every other caller
// should use startReading() instead.
func (r *imageStore) startWritingWithReload(canReload bool) error {
	r.lockfile.Lock()
	succeeded := false
	defer func() {
		if !succeeded {
			r.lockfile.Unlock()
		}
	}()

	if canReload {
		if err := r.reloadIfChanged(true); err != nil {
			return err
		}
	}

	succeeded = true
	return nil
}

// startWriting makes sure the store is fresh, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
func (r *imageStore) startWriting() error {
	return r.startWritingWithReload(true)
}

// stopWriting releases locks obtained by startWriting.
func (r *imageStore) stopWriting() {
	r.lockfile.Unlock()
}

// startReadingWithReload makes sure the store is fresh if canReload, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
//
// This is an internal implementation detail of imageStore construction, every other caller
// should use startReading() instead.
func (r *imageStore) startReadingWithReload(canReload bool) error {
	r.lockfile.RLock()
	succeeded := false
	defer func() {
		if !succeeded {
			r.lockfile.Unlock()
		}
	}()

	if canReload {
		if err := r.reloadIfChanged(false); err != nil {
			return err
		}
	}

	succeeded = true
	return nil
}

// startReading makes sure the store is fresh, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
func (r *imageStore) startReading() error {
	return r.startReadingWithReload(true)
}

// stopReading releases locks obtained by startReading.
func (r *imageStore) stopReading() {
	r.lockfile.Unlock()
}

// reloadIfChanged reloads the contents of the store from disk if it is changed.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
func (r *imageStore) reloadIfChanged(lockedForWriting bool) error {
	r.loadMut.Lock()
	defer r.loadMut.Unlock()

	modified, err := r.lockfile.Modified()
	if err != nil {
		return err
	}
	if modified {
		return r.load(lockedForWriting)
	}
	return nil
}

func (r *imageStore) Images() ([]Image, error) {
	images := make([]Image, len(r.images))
	for i := range r.images {
		images[i] = *copyImage(r.images[i])
	}
	return images, nil
}

func (r *imageStore) imagespath() string {
	return filepath.Join(r.dir, "images.json")
}

func (r *imageStore) datadir(id string) string {
	return filepath.Join(r.dir, id)
}

func (r *imageStore) datapath(id, key string) string {
	return filepath.Join(r.datadir(id), makeBigDataBaseName(key))
}

// bigDataNameIsManifest determines if a big data item with the specified name
// is considered to be representative of the image, in that its digest can be
// said to also be the image's digest.  Currently, if its name is, or begins
// with, "manifest", we say that it is.
func bigDataNameIsManifest(name string) bool {
	return strings.HasPrefix(name, ImageDigestManifestBigDataNamePrefix)
}

// recomputeDigests takes a fixed digest and a name-to-digest map and builds a
// list of the unique values that would identify the image.
func (i *Image) recomputeDigests() error {
	validDigests := make([]digest.Digest, 0, len(i.BigDataDigests)+1)
	digests := make(map[digest.Digest]struct{})
	if i.Digest != "" {
		if err := i.Digest.Validate(); err != nil {
			return fmt.Errorf("validating image digest %q: %w", string(i.Digest), err)
		}
		digests[i.Digest] = struct{}{}
		validDigests = append(validDigests, i.Digest)
	}
	for name, digest := range i.BigDataDigests {
		if !bigDataNameIsManifest(name) {
			continue
		}
		if err := digest.Validate(); err != nil {
			return fmt.Errorf("validating digest %q for big data item %q: %w", string(digest), name, err)
		}
		// Deduplicate the digest values.
		if _, known := digests[digest]; !known {
			digests[digest] = struct{}{}
			validDigests = append(validDigests, digest)
		}
	}
	if i.Digest == "" && len(validDigests) > 0 {
		i.Digest = validDigests[0]
	}
	i.Digests = validDigests
	return nil
}

// load reloads the contents of the store from disk.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
func (r *imageStore) load(lockedForWriting bool) error {
	rpath := r.imagespath()
	data, err := os.ReadFile(rpath)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	images := []*Image{}
	if len(data) != 0 {
		if err := json.Unmarshal(data, &images); err != nil {
			return fmt.Errorf("loading %q: %w", rpath, err)
		}
	}
	idlist := make([]string, 0, len(images))
	ids := make(map[string]*Image)
	names := make(map[string]*Image)
	digests := make(map[digest.Digest][]*Image)
	var errorToResolveBySaving error // == nil
	for n, image := range images {
		ids[image.ID] = images[n]
		idlist = append(idlist, image.ID)
		for _, name := range image.Names {
			if conflict, ok := names[name]; ok {
				r.removeName(conflict, name)
				errorToResolveBySaving = ErrDuplicateImageNames
			}
		}
		// Compute the digest list.
		if err := image.recomputeDigests(); err != nil {
			return fmt.Errorf("computing digests for image with ID %q (%v): %w", image.ID, image.Names, err)
		}
		for _, name := range image.Names {
			names[name] = image
		}
		for _, digest := range image.Digests {
			list := digests[digest]
			digests[digest] = append(list, image)
		}
		image.ReadOnly = !r.lockfile.IsReadWrite()
	}

	if errorToResolveBySaving != nil && (!r.lockfile.IsReadWrite() || !lockedForWriting) {
		// Eventually, the callers should be modified to retry with a write lock if IsReadWrite && !lockedForWriting, instead.
		return errorToResolveBySaving
	}
	r.images = images
	r.idindex = truncindex.NewTruncIndex(idlist) // Invalid values in idlist are ignored: they are not a reason to refuse processing the whole store.
	r.byid = ids
	r.byname = names
	r.bydigest = digests
	if errorToResolveBySaving != nil {
		return r.Save()
	}
	return nil
}

// Save saves the contents of the store to disk.  It should be called with
// the lock held, locked for writing.
func (r *imageStore) Save() error {
	if !r.lockfile.IsReadWrite() {
		return fmt.Errorf("not allowed to modify the image store at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
	}
	r.lockfile.AssertLockedForWriting()
	rpath := r.imagespath()
	if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
		return err
	}
	jdata, err := json.Marshal(&r.images)
	if err != nil {
		return err
	}
	if err := ioutils.AtomicWriteFile(rpath, jdata, 0600); err != nil {
		return err
	}
	return r.lockfile.Touch()
}

func newImageStore(dir string) (rwImageStore, error) {
	if err := os.MkdirAll(dir, 0700); err != nil {
		return nil, err
	}
	lockfile, err := GetLockfile(filepath.Join(dir, "images.lock"))
	if err != nil {
		return nil, err
	}
	istore := imageStore{
		lockfile: lockfile,
		dir:      dir,
		images:   []*Image{},
		byid:     make(map[string]*Image),
		byname:   make(map[string]*Image),
		bydigest: make(map[digest.Digest][]*Image),
	}
	if err := istore.startWritingWithReload(false); err != nil {
		return nil, err
	}
	defer istore.stopWriting()
	if err := istore.load(true); err != nil {
		return nil, err
	}
	return &istore, nil
}

func newROImageStore(dir string) (roImageStore, error) {
	lockfile, err := GetROLockfile(filepath.Join(dir, "images.lock"))
	if err != nil {
		return nil, err
	}
	istore := imageStore{
		lockfile: lockfile,
		dir:      dir,
		images:   []*Image{},
		byid:     make(map[string]*Image),
		byname:   make(map[string]*Image),
		bydigest: make(map[digest.Digest][]*Image),
	}
	if err := istore.startReadingWithReload(false); err != nil {
		return nil, err
	}
	defer istore.stopReading()
	if err := istore.load(false); err != nil {
		return nil, err
	}
	return &istore, nil
}

func (r *imageStore) lookup(id string) (*Image, bool) {
	if image, ok := r.byid[id]; ok {
		return image, ok
	} else if image, ok := r.byname[id]; ok {
		return image, ok
	} else if longid, err := r.idindex.Get(id); err == nil {
		image, ok := r.byid[longid]
		return image, ok
	}
	return nil, false
}

func (r *imageStore) ClearFlag(id string, flag string) error {
	if !r.lockfile.IsReadWrite() {<commit_msg>Clean up duplicate image names in readers instead of failing<commit_after>package storage

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/containers/storage/pkg/ioutils"
	"github.com/containers/storage/pkg/stringid"
	"github.com/containers/storage/pkg/stringutils"
	"github.com/containers/storage/pkg/truncindex"
	digest "github.com/opencontainers/go-digest"
)

const (
	// ImageDigestManifestBigDataNamePrefix is a prefix of big data item
	// names which we consider to be manifests, used for computing a
	// "digest" value for the image as a whole, by which we can locate the
	// image later.
	ImageDigestManifestBigDataNamePrefix = "manifest"
	// ImageDigestBigDataKey is provided for compatibility with older
	// versions of the image library.  It will be removed in the future.
	ImageDigestBigDataKey = "manifest"
)

// An Image is a reference to a layer and an associated metadata string.
type Image struct {
	// ID is either one which was specified at create-time, or a random
	// value which was generated by the library.
	ID string `json:"id"`

	// Digest is a digest value that we can use to locate the image, if one
	// was specified at creation-time.
	Digest digest.Digest `json:"digest,omitempty"`

	// Digests is a list of digest values of the image's manifests, and
	// possibly a manually-specified value, that we can use to locate the
	// image.  If Digest is set, its value is also in this list.
	Digests []digest.Digest `json:"-"`

	// Names is an optional set of user-defined convenience values.  The
	// image can be referred to by its ID or any of its names.  Names are
	// unique among images, and are often the text representation of tagged
	// or canonical references.
	Names []string `json:"names,omitempty"`

	// NamesHistory is an optional set of Names the image had in the past. The
	// contained names are free from any duplicates, whereas the newest entry
	// is the first one.
	NamesHistory []string `json:"names-history,omitempty"`

	// TopLayer is the ID of the topmost layer of the image itself, if the
	// image contains one or more layers.  Multiple images can refer to the
	// same top layer.
	TopLayer string `json:"layer,omitempty"`

	// MappedTopLayers are the IDs of alternate versions of the top layer
	// which have the same contents and parent, and which differ from
	// TopLayer only in which ID mappings they use.  When the image is
	// to be removed, they should be removed before the TopLayer, as the
	// graph driver may depend on that.
	MappedTopLayers []string `json:"mapped-layers,omitempty"`

	// Metadata is data we keep for the convenience of the caller.  It is not
	// expected to be large, since it is kept in memory.
	Metadata string `json:"metadata,omitempty"`

	// BigDataNames is a list of names of data items that we keep for the
	// convenience of the caller.  They can be large, and are only in
	// memory when being read from or written to disk.
	BigDataNames []string `json:"big-data-names,omitempty"`

	// BigDataSizes maps the names in BigDataNames to the sizes of the data
	// that has been stored, if they're known.
	BigDataSizes map[string]int64 `json:"big-data-sizes,omitempty"`

	// BigDataDigests maps the names in BigDataNames to the digests of the
	// data that has been stored, if they're known.
	BigDataDigests map[string]digest.Digest `json:"big-data-digests,omitempty"`

	// Created is the datestamp for when this image was created.  Older
	// versions of the library did not track this information, so callers
	// will likely want to use the IsZero() method to verify that a value
	// is set before using it.
	Created time.Time `json:"created,omitempty"`

	// ReadOnly is true if this image resides in a read-only layer store.
	ReadOnly bool `json:"-"`

	Flags map[string]interface{} `json:"flags,omitempty"`
}

// roImageStore provides bookkeeping for information about Images.
type roImageStore interface {
	roMetadataStore
	roBigDataStore

	// startReading makes sure the store is fresh, and locks it for reading.
	// If this succeeds, the caller MUST call stopReading().
	startReading() error

	// stopReading releases locks obtained by startReading.
	stopReading()

	// Exists checks if there is an image with the given ID or name.
	Exists(id string) bool

	// Get retrieves information about an image given an ID or name.
	Get(id string) (*Image, error)

	// Images returns a slice enumerating the known images.
	Images() ([]Image, error)

	// ByDigest returns a slice enumerating the images which have either an
	// explicitly-set digest, or a big data item with a name that starts
	// with ImageDigestManifestBigDataNamePrefix, which matches the
	// specified digest.
	ByDigest(d digest.Digest) ([]*Image, error)
}

// rwImageStore provides bookkeeping for information about Images.
type rwImageStore interface {
	roImageStore
	rwMetadataStore
	rwImageBigDataStore
	flaggableStore

	// startWriting makes sure the store is fresh, and locks it for writing.
	// If this succeeds, the caller MUST call stopWriting().
	startWriting() error

	// stopWriting releases locks obtained by startWriting.
	stopWriting()

	// Create creates an image that has a specified ID (or a random one) and
	// optional names, using the specified layer as its topmost (hopefully
	// read-only) layer.  That layer can be referenced by multiple images.
	Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (*Image, error)

	// updateNames modifies names associated with an image based on (op, names).
	// The values are expected to be valid normalized
	// named image references.
	updateNames(id string, names []string, op updateNameOperation) error

	// Delete removes the record of the image.
	Delete(id string) error

	// Wipe removes records of all images.
	Wipe() error
}

type imageStore struct {
	lockfile Locker // lockfile.IsReadWrite can be used to distinguish between read-write and read-only image stores.
	dir      string
	images   []*Image
	idindex  *truncindex.TruncIndex
	byid     map[string]*Image
	byname   map[string]*Image
	bydigest map[digest.Digest][]*Image
	loadMut  sync.Mutex
}

func copyImage(i *Image) *Image {
	return &Image{
		ID:              i.ID,
		Digest:          i.Digest,
		Digests:         copyDigestSlice(i.Digests),
		Names:           copyStringSlice(i.Names),
		NamesHistory:    copyStringSlice(i.NamesHistory),
		TopLayer:        i.TopLayer,
		MappedTopLayers: copyStringSlice(i.MappedTopLayers),
		Metadata:        i.Metadata,
		BigDataNames:    copyStringSlice(i.BigDataNames),
		BigDataSizes:    copyStringInt64Map(i.BigDataSizes),
		BigDataDigests:  copyStringDigestMap(i.BigDataDigests),
		Created:         i.Created,
		ReadOnly:        i.ReadOnly,
		Flags:           copyStringInterfaceMap(i.Flags),
	}
}

func copyImageSlice(slice []*Image) []*Image {
	if len(slice) > 0 {
		cp := make([]*Image, len(slice))
		for i := range slice {
			cp[i] = copyImage(slice[i])
		}
		return cp
	}
	return nil
}

// startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
//
// This is an internal implementation detail of imageStore construction, every other caller
// should use startReading() instead.
func (r *imageStore) startWritingWithReload(canReload bool) error {
	r.lockfile.Lock()
	succeeded := false
	defer func() {
		if !succeeded {
			r.lockfile.Unlock()
		}
	}()

	if canReload {
		if _, err := r.reloadIfChanged(true); err != nil {
			return err
		}
	}

	succeeded = true
	return nil
}

// startWriting makes sure the store is fresh, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
func (r *imageStore) startWriting() error {
	return r.startWritingWithReload(true)
}

// stopWriting releases locks obtained by startWriting.
func (r *imageStore) stopWriting() {
	r.lockfile.Unlock()
}

// startReadingWithReload makes sure the store is fresh if canReload, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
//
// This is an internal implementation detail of imageStore construction, every other caller
// should use startReading() instead.
func (r *imageStore) startReadingWithReload(canReload bool) error {
	r.lockfile.RLock()
	unlockFn := r.lockfile.Unlock // A function to call to clean up, or nil
	defer func() {
		if unlockFn != nil {
			unlockFn()
		}
	}()

	if canReload {
		if tryLockedForWriting, err := r.reloadIfChanged(false); err != nil {
			if !tryLockedForWriting {
				return err
			}
			unlockFn()
			unlockFn = nil

			r.lockfile.Lock()
			unlockFn = r.lockfile.Unlock
			if _, err := r.load(true); err != nil {
				return err
			}
			unlockFn()
			unlockFn = nil

			r.lockfile.RLock()
			unlockFn = r.lockfile.Unlock
			// We need to check for a reload reload once more because the on-disk state could have been modified
			// after we released the lock.
			// If that, _again_, finds inconsistent state, just give up.
			// We could, plausibly, retry a few times, but that inconsistent state (duplicate image names)
			// shouldn’t be saved (by correct implementations) in the first place.
			if _, err := r.reloadIfChanged(false); err != nil {
				return fmt.Errorf("(even after successfully cleaning up once:) %w", err)
			}
		}
	}

	unlockFn = nil
	return nil
}

// startReading makes sure the store is fresh, and locks it for reading.
// If this succeeds, the caller MUST call stopReading().
func (r *imageStore) startReading() error {
	return r.startReadingWithReload(true)
}

// stopReading releases locks obtained by startReading.
func (r *imageStore) stopReading() {
	r.lockfile.Unlock()
}

// reloadIfChanged reloads the contents of the store from disk if it is changed.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
//
// If !lockedForWriting and this function fails, the return value indicates whether
// retrying with lockedForWriting could succeed. In that case the caller MUST
// call load(), not reloadIfChanged() (because the “if changed” state will not
// be detected again).
func (r *imageStore) reloadIfChanged(lockedForWriting bool) (bool, error) {
	r.loadMut.Lock()
	defer r.loadMut.Unlock()

	modified, err := r.lockfile.Modified()
	if err != nil {
		return false, err
	}
	if modified {
		return r.load(lockedForWriting)
	}
	return false, nil
}

func (r *imageStore) Images() ([]Image, error) {
	images := make([]Image, len(r.images))
	for i := range r.images {
		images[i] = *copyImage(r.images[i])
	}
	return images, nil
}

func (r *imageStore) imagespath() string {
	return filepath.Join(r.dir, "images.json")
}

func (r *imageStore) datadir(id string) string {
	return filepath.Join(r.dir, id)
}

func (r *imageStore) datapath(id, key string) string {
	return filepath.Join(r.datadir(id), makeBigDataBaseName(key))
}

// bigDataNameIsManifest determines if a big data item with the specified name
// is considered to be representative of the image, in that its digest can be
// said to also be the image's digest.  Currently, if its name is, or begins
// with, "manifest", we say that it is.
func bigDataNameIsManifest(name string) bool {
	return strings.HasPrefix(name, ImageDigestManifestBigDataNamePrefix)
}

// recomputeDigests takes a fixed digest and a name-to-digest map and builds a
// list of the unique values that would identify the image.
func (i *Image) recomputeDigests() error {
	validDigests := make([]digest.Digest, 0, len(i.BigDataDigests)+1)
	digests := make(map[digest.Digest]struct{})
	if i.Digest != "" {
		if err := i.Digest.Validate(); err != nil {
			return fmt.Errorf("validating image digest %q: %w", string(i.Digest), err)
		}
		digests[i.Digest] = struct{}{}
		validDigests = append(validDigests, i.Digest)
	}
	for name, digest := range i.BigDataDigests {
		if !bigDataNameIsManifest(name) {
			continue
		}
		if err := digest.Validate(); err != nil {
			return fmt.Errorf("validating digest %q for big data item %q: %w", string(digest), name, err)
		}
		// Deduplicate the digest values.
		if _, known := digests[digest]; !known {
			digests[digest] = struct{}{}
			validDigests = append(validDigests, digest)
		}
	}
	if i.Digest == "" && len(validDigests) > 0 {
		i.Digest = validDigests[0]
	}
	i.Digests = validDigests
	return nil
}

// load reloads the contents of the store from disk.
//
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
//
// If !lockedForWriting and this function fails, the return value indicates whether
// retrying with lockedForWriting could succeed.
func (r *imageStore) load(lockedForWriting bool) (bool, error) {
	rpath := r.imagespath()
	data, err := os.ReadFile(rpath)
	if err != nil && !os.IsNotExist(err) {
		return false, err
	}

	images := []*Image{}
	if len(data) != 0 {
		if err := json.Unmarshal(data, &images); err != nil {
			return false, fmt.Errorf("loading %q: %w", rpath, err)
		}
	}
	idlist := make([]string, 0, len(images))
	ids := make(map[string]*Image)
	names := make(map[string]*Image)
	digests := make(map[digest.Digest][]*Image)
	var errorToResolveBySaving error // == nil
	for n, image := range images {
		ids[image.ID] = images[n]
		idlist = append(idlist, image.ID)
		for _, name := range image.Names {
			if conflict, ok := names[name]; ok {
				r.removeName(conflict, name)
				errorToResolveBySaving = ErrDuplicateImageNames
			}
		}
		// Compute the digest list.
		if err := image.recomputeDigests(); err != nil {
			return false, fmt.Errorf("computing digests for image with ID %q (%v): %w", image.ID, image.Names, err)
		}
		for _, name := range image.Names {
			names[name] = image
		}
		for _, digest := range image.Digests {
			list := digests[digest]
			digests[digest] = append(list, image)
		}
		image.ReadOnly = !r.lockfile.IsReadWrite()
	}

	if errorToResolveBySaving != nil {
		if !r.lockfile.IsReadWrite() {
			return false, errorToResolveBySaving
		}
		if !lockedForWriting {
			return true, errorToResolveBySaving
		}
	}
	r.images = images
	r.idindex = truncindex.NewTruncIndex(idlist) // Invalid values in idlist are ignored: they are not a reason to refuse processing the whole store.
	r.byid = ids
	r.byname = names
	r.bydigest = digests
	if errorToResolveBySaving != nil {
		return false, r.Save()
	}
	return false, nil
}

// Save saves the contents of the store to disk.  It should be called with
// the lock held, locked for writing.
func (r *imageStore) Save() error {
	if !r.lockfile.IsReadWrite() {
		return fmt.Errorf("not allowed to modify the image store at %q: %w", r.imagespath(), ErrStoreIsReadOnly)
	}
	r.lockfile.AssertLockedForWriting()
	rpath := r.imagespath()
	if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
		return err
	}
	jdata, err := json.Marshal(&r.images)
	if err != nil {
		return err
	}
	if err := ioutils.AtomicWriteFile(rpath, jdata, 0600); err != nil {
		return err
	}
	return r.lockfile.Touch()
}

func newImageStore(dir string) (rwImageStore, error) {
	if err := os.MkdirAll(dir, 0700); err != nil {
		return nil, err
	}
	lockfile, err := GetLockfile(filepath.Join(dir, "images.lock"))
	if err != nil {
		return nil, err
	}
	istore := imageStore{
		lockfile: lockfile,
		dir:      dir,
		images:   []*Image{},
		byid:     make(map[string]*Image),
		byname:   make(map[string]*Image),
		bydigest: make(map[digest.Digest][]*Image),
	}
	if err := istore.startWritingWithReload(false); err != nil {
		return nil, err
	}
	defer istore.stopWriting()
	if _, err := istore.load(true); err != nil {
		return nil, err
	}
	return &istore, nil
}

func newROImageStore(dir string) (roImageStore, error) {
	lockfile, err := GetROLockfile(filepath.Join(dir, "images.lock"))
	if err != nil {
		return nil, err
	}
	istore := imageStore{
		lockfile: lockfile,
		dir:      dir,
		images:   []*Image{},
		byid:     make(map[string]*Image),
		byname:   make(map[string]*Image),
		bydigest: make(map[digest.Digest][]*Image),
	}
	if err := istore.startReadingWithReload(false); err != nil {
		return nil, err
	}
	defer istore.stopReading()
	if _, err := istore.load(false); err != nil {
		return nil, err
	}
	return &istore, nil
}

func (r *imageStore) lookup(id string) (*Image, bool) {
	if image, ok := r.byid[id]; ok {
		return image, ok
	} else if image, ok := r.byname[id]; ok {
		return image, ok
	} else if longid, err := r.idindex.Get(id); err == nil {
		image, ok := r.byid[longid]
		return image, ok
	}
	return nil, false
}

func (r *imageStore) ClearFlag(id string, flag string) error {
	if !r.lockfile.IsReadWrite() {
{'fraction_non_alphanumeric': 0.10928978875446624, 'fraction_numerical': 0.001441735096846988, 'mean_word_length': 3.7325719371106496, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4681803', 'n_tokens_mistral': 10619, 'n_tokens_neox': 10024, 'n_words': 3612}