<commit_before>package tarfile

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/containers/image/v5/docker/reference"
	"github.com/containers/image/v5/manifest"
	"github.com/containers/image/v5/types"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// Writer allows creating a (docker save)-formatted tar archive containing one or more images.
type Writer struct {
	writer io.Writer
	tar    *tar.Writer
	// Other state.
	blobs            map[digest.Digest]types.BlobInfo // list of already-sent blobs
	manifest         []ManifestItem
	manifestByConfig map[digest.Digest]int // A map from config digest to an entry index in manifest above.
}

// NewWriter returns a Writer for the specified io.Writer.
// The caller must eventually call .Close() on the returned object to create a valid archive.
func NewWriter(dest io.Writer) *Writer {
	return &Writer{
		writer:           dest,
		tar:              tar.NewWriter(dest),
		blobs:            make(map[digest.Digest]types.BlobInfo),
		manifestByConfig: map[digest.Digest]int{},
	}
}

// tryReusingBlob checks whether the transport already contains, a blob, and if so, returns its metadata.
// info.Digest must not be empty.
// If the blob has been succesfully reused, returns (true, info, nil); info must contain at least a digest and size.
// If the transport can not reuse the requested blob, tryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
func (w *Writer) tryReusingBlob(info types.BlobInfo) (bool, types.BlobInfo, error) {
	if info.Digest == "" {
		return false, types.BlobInfo{}, errors.Errorf("Can not check for a blob with unknown digest")
	}
	if blob, ok := w.blobs[info.Digest]; ok {
		return true, types.BlobInfo{Digest: info.Digest, Size: blob.Size}, nil
	}
	return false, types.BlobInfo{}, nil
}

// recordBlob records metadata of a recorded blob, which must contain at least a digest and size.
func (w *Writer) recordBlob(info types.BlobInfo) {
	w.blobs[info.Digest] = info
}

// createSingleLegacyLayer writes legacy VERSION and configuration files for a single layer
func (w *Writer) createSingleLegacyLayer(layerID string, layerDigest digest.Digest, configBytes []byte) error {
	// Create a symlink for the legacy format, where there is one subdirectory per layer ("image").
	// See also the comment in physicalLayerPath.
	physicalLayerPath := w.physicalLayerPath(layerDigest)
	if err := w.sendSymlink(filepath.Join(layerID, legacyLayerFileName), filepath.Join("..", physicalLayerPath)); err != nil {
		return errors.Wrap(err, "Error creating layer symbolic link")
	}

	b := []byte("1.0")
	if err := w.sendBytes(filepath.Join(layerID, legacyVersionFileName), b); err != nil {
		return errors.Wrap(err, "Error writing VERSION file")
	}

	if err := w.sendBytes(filepath.Join(layerID, legacyConfigFileName), configBytes); err != nil {
		return errors.Wrap(err, "Error writing config json file")
	}
	return nil
}

// writeLegacyLayerMetadata writes legacy VERSION and configuration files for all layers
func (w *Writer) writeLegacyLayerMetadata(layerDescriptors []manifest.Schema2Descriptor, configBytes []byte) (lastLayerID string, err error) {
	var chainID digest.Digest
	lastLayerID = ""
	for i, l := range layerDescriptors {
		// This chainID value matches the computation in docker/docker/layer.CreateChainID …
		if chainID == "" {
			chainID = l.Digest
		} else {
			chainID = digest.Canonical.FromString(chainID.String() + " " + l.Digest.String())
		}
		// … but note that this image ID does not match docker/docker/image/v1.CreateID. At least recent
		// versions allocate new IDs on load, as long as the IDs we use are unique / cannot loop.
		//
		// Overall, the goal of computing a digest dependent on the full history is to avoid reusing an image ID
		// (and possibly creating a loop in the "parent" links) if a layer with the same DiffID appears two or more
		// times in layersDescriptors.  The ChainID values are sufficient for this, the v1.CreateID computation
		// which also mixes in the full image configuration seems unnecessary, at least as long as we are storing
		// only a single image per tarball, i.e. all DiffID prefixes are unique (can’t differ only with
		// configuration).
		layerID := chainID.Hex()

		// The legacy format requires a config file per layer
		layerConfig := make(map[string]interface{})
		layerConfig["id"] = layerID

		// The root layer doesn't have any parent
		if lastLayerID != "" {
			layerConfig["parent"] = lastLayerID
		}
		// The top layer configuration file is generated by using subpart of the image configuration
		if i == len(layerDescriptors)-1 {
			var config map[string]*json.RawMessage
			err := json.Unmarshal(configBytes, &config)
			if err != nil {
				return "", errors.Wrap(err, "Error unmarshaling config")
			}
			for _, attr := range [7]string{"architecture", "config", "container", "container_config", "created", "docker_version", "os"} {
				layerConfig[attr] = config[attr]
			}
		}
		configBytes, err := json.Marshal(layerConfig)
		if err != nil {
			return "", errors.Wrap(err, "Error marshaling layer config")
		}

		if err := w.createSingleLegacyLayer(layerID, l.Digest, configBytes); err != nil {
			return "", err
		}

		lastLayerID = layerID
	}
	return lastLayerID, nil
}

func (w *Writer) createRepositoriesFile(rootLayerID string, repoTags []reference.NamedTagged) error {
	repositories := map[string]map[string]string{}
	for _, repoTag := range repoTags {
		if val, ok := repositories[repoTag.Name()]; ok {
			val[repoTag.Tag()] = rootLayerID
		} else {
			repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): rootLayerID}
		}
	}

	b, err := json.Marshal(repositories)
	if err != nil {
		return errors.Wrap(err, "Error marshaling repositories")
	}<commit_msg>Move legacy layer ID computation to a bit later<commit_after>package tarfile

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/containers/image/v5/docker/reference"
	"github.com/containers/image/v5/manifest"
	"github.com/containers/image/v5/types"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// Writer allows creating a (docker save)-formatted tar archive containing one or more images.
type Writer struct {
	writer io.Writer
	tar    *tar.Writer
	// Other state.
	blobs            map[digest.Digest]types.BlobInfo // list of already-sent blobs
	manifest         []ManifestItem
	manifestByConfig map[digest.Digest]int // A map from config digest to an entry index in manifest above.
}

// NewWriter returns a Writer for the specified io.Writer.
// The caller must eventually call .Close() on the returned object to create a valid archive.
func NewWriter(dest io.Writer) *Writer {
	return &Writer{
		writer:           dest,
		tar:              tar.NewWriter(dest),
		blobs:            make(map[digest.Digest]types.BlobInfo),
		manifestByConfig: map[digest.Digest]int{},
	}
}

// tryReusingBlob checks whether the transport already contains, a blob, and if so, returns its metadata.
// info.Digest must not be empty.
// If the blob has been succesfully reused, returns (true, info, nil); info must contain at least a digest and size.
// If the transport can not reuse the requested blob, tryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
func (w *Writer) tryReusingBlob(info types.BlobInfo) (bool, types.BlobInfo, error) {
	if info.Digest == "" {
		return false, types.BlobInfo{}, errors.Errorf("Can not check for a blob with unknown digest")
	}
	if blob, ok := w.blobs[info.Digest]; ok {
		return true, types.BlobInfo{Digest: info.Digest, Size: blob.Size}, nil
	}
	return false, types.BlobInfo{}, nil
}

// recordBlob records metadata of a recorded blob, which must contain at least a digest and size.
func (w *Writer) recordBlob(info types.BlobInfo) {
	w.blobs[info.Digest] = info
}

// createSingleLegacyLayer writes legacy VERSION and configuration files for a single layer
func (w *Writer) createSingleLegacyLayer(layerID string, layerDigest digest.Digest, configBytes []byte) error {
	// Create a symlink for the legacy format, where there is one subdirectory per layer ("image").
	// See also the comment in physicalLayerPath.
	physicalLayerPath := w.physicalLayerPath(layerDigest)
	if err := w.sendSymlink(filepath.Join(layerID, legacyLayerFileName), filepath.Join("..", physicalLayerPath)); err != nil {
		return errors.Wrap(err, "Error creating layer symbolic link")
	}

	b := []byte("1.0")
	if err := w.sendBytes(filepath.Join(layerID, legacyVersionFileName), b); err != nil {
		return errors.Wrap(err, "Error writing VERSION file")
	}

	if err := w.sendBytes(filepath.Join(layerID, legacyConfigFileName), configBytes); err != nil {
		return errors.Wrap(err, "Error writing config json file")
	}
	return nil
}

// writeLegacyLayerMetadata writes legacy VERSION and configuration files for all layers
func (w *Writer) writeLegacyLayerMetadata(layerDescriptors []manifest.Schema2Descriptor, configBytes []byte) (lastLayerID string, err error) {
	var chainID digest.Digest
	lastLayerID = ""
	for i, l := range layerDescriptors {
		// The legacy format requires a config file per layer
		layerConfig := make(map[string]interface{})

		// The root layer doesn't have any parent
		if lastLayerID != "" {
			layerConfig["parent"] = lastLayerID
		}
		// The top layer configuration file is generated by using subpart of the image configuration
		if i == len(layerDescriptors)-1 {
			var config map[string]*json.RawMessage
			err := json.Unmarshal(configBytes, &config)
			if err != nil {
				return "", errors.Wrap(err, "Error unmarshaling config")
			}
			for _, attr := range [7]string{"architecture", "config", "container", "container_config", "created", "docker_version", "os"} {
				layerConfig[attr] = config[attr]
			}
		}

		// This chainID value matches the computation in docker/docker/layer.CreateChainID …
		if chainID == "" {
			chainID = l.Digest
		} else {
			chainID = digest.Canonical.FromString(chainID.String() + " " + l.Digest.String())
		}
		// … but note that this image ID does not match docker/docker/image/v1.CreateID. At least recent
		// versions allocate new IDs on load, as long as the IDs we use are unique / cannot loop.
		//
		// Overall, the goal of computing a digest dependent on the full history is to avoid reusing an image ID
		// (and possibly creating a loop in the "parent" links) if a layer with the same DiffID appears two or more
		// times in layersDescriptors.  The ChainID values are sufficient for this, the v1.CreateID computation
		// which also mixes in the full image configuration seems unnecessary, at least as long as we are storing
		// only a single image per tarball, i.e. all DiffID prefixes are unique (can’t differ only with
		// configuration).
		layerID := chainID.Hex()
		layerConfig["id"] = layerID

		configBytes, err := json.Marshal(layerConfig)
		if err != nil {
			return "", errors.Wrap(err, "Error marshaling layer config")
		}

		if err := w.createSingleLegacyLayer(layerID, l.Digest, configBytes); err != nil {
			return "", err
		}

		lastLayerID = layerID
	}
	return lastLayerID, nil
}

func (w *Writer) createRepositoriesFile(rootLayerID string, repoTags []reference.NamedTagged) error {
	repositories := map[string]map[string]string{}
	for _, repoTag := range repoTags {
		if val, ok := repositories[repoTag.Name()]; ok {
			val[repoTag.Tag()] = rootLayerID
		} else {
			repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): rootLayerID}
		}
	}

	b, err := json.Marshal(repositories)
	if err != nil {
		return errors.Wrap(err, "Error marshaling repositories")
	}
{'fraction_non_alphanumeric': 0.11147874777297022, 'fraction_numerical': 0.0016967845931958937, 'mean_word_length': 3.9949152542372883, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5980773', 'n_tokens_mistral': 3764, 'n_tokens_neox': 3539, 'n_words': 1226}