<commit_before>package libsquash

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/winchman/libsquash/tarball"
)

var uuidRegex = regexp.MustCompile("^[a-f0-9]{64}$")

func (e *export) IngestImageMetadata(tarstream io.Reader) error {
	if err := tarball.Walk(tarstream, func(t *tarball.TarFile) error {
		switch Type(t) {
		case IGNORE:
			return nil
		case REPOSITORIES:
			if err := json.NewDecoder(t.Stream).Decode(&e.Repositories); err != nil {
				return err
			}
			// Export may have multiple branches with the same parent. Abort.
			for _, v := range e.Repositories {
				commits := map[string]string{}
				for tag, commit := range *v {
					commits[commit] = tag
				}
				if len(commits) > 1 {
					return errorMultipleBranchesSameParent
				}
			}
		case JSON:
			uuid := t.NameParts()[0]
			if e.Layers[uuid] == nil {
				e.Layers[uuid] = &layer{}
			}
			e.Layers[uuid].JSONHeader = t.Header
			if err := json.NewDecoder(t.Stream).Decode(&e.Layers[uuid].LayerConfig); err != nil {
				return err
			}
		case LAYER_TAR:
			uuid := t.NameParts()[0]
			if e.Layers[uuid] == nil {
				e.Layers[uuid] = &layer{}
			}
			if err := tarball.Walk(t.Stream, func(tf *tarball.TarFile) error {
				filePath := nameWithoutWhiteoutPrefix(tf.Name())
				if e.fileToLayers[filePath] == nil {
					e.fileToLayers[filePath] = []fileLoc{}
				}
				foundWhiteout := isWhiteout(tf.Name())
				e.fileToLayers[filePath] = append(e.fileToLayers[filePath], fileLoc{
					uuid:     uuid,
					whiteout: foundWhiteout,
				})

				if foundWhiteout {
					e.whiteouts = append(e.whiteouts, whiteoutFile{
						prefix: filePath,
						uuid:   uuid,
					})
				}
				return nil
			}); err != nil {
				return err
			}
		}
		return nil
	}); err != nil {
		return err
	}

	return e.populateFileData()
}

func (e *export) populateFileData() error {

	e.start = e.FirstSquash()

	// Can't find a previously squashed layer, default to root
	if e.start == nil {
		e.start = e.Root()
	}

	if e.start == nil {
		return errorNoFROM
	}

	index := 0
	current := e.start
	orderMap := map[string]int{}
	for {
		orderMap[current.LayerConfig.ID] = index
		index++
		current = e.ChildOf(current.LayerConfig.ID)
		if current == nil {
			break
		}
	}

	for path, fileLocs := range e.fileToLayers {
		if len(fileLocs) > 0 {
			greatest := fileLocs[0]
			for _, loc := range fileLocs {
				if orderMap[loc.uuid] > orderMap[greatest.uuid] {
					greatest = loc
				}
			}
			if e.layerToFiles[greatest.uuid] == nil {
				e.layerToFiles[greatest.uuid] = map[string]bool{}
			}

			// if name matches whiteout prefix and the whiteout file is found
			// in a layer that is >= greatest.uuid, skip
			uuidContainingWhiteout, matches := matchesWhiteout(path, e.whiteouts)
			if (matches && orderMap[uuidContainingWhiteout] >= orderMap[greatest.uuid]) || greatest.whiteout {
				delete(e.layerToFiles[greatest.uuid], path)
			} else {
				e.layerToFiles[greatest.uuid][path] = true
			}
		}
	}

	return nil
}

func (e *export) SquashLayers(into, from *layer, tarstream io.Reader, outstream io.Writer) (imageID string, err error) {
	tempfile, err := ioutil.TempFile("", "libsquash")
	if err != nil {
		return "", err
	}

	defer func() {
		_ = tempfile.Close()
		_ = os.RemoveAll(tempfile.Name())
	}()

	var squashLayerTarWriter = tar.NewWriter(tempfile)

	if err = tarball.Walk(tarstream, func(t *tarball.TarFile) error {
		nameParts := t.NameParts()
		switch Type(t) {
		case DIRECTORY:
			uuidPart := nameParts[0]
			e.Layers[uuidPart].DirHeader = t.Header
		case LAYER_TAR:
			uuidPart := nameParts[0]
			e.Layers[uuidPart].LayerTarHeader = t.Header
			if err := tarball.Walk(t.Stream, func(tf *tarball.TarFile) error {
				filePath := nameWithoutWhiteoutPrefix(tf.Name())
				if e.layerToFiles[uuidPart][filePath] {
					squashLayerTarWriter.WriteHeader(tf.Header)
					if _, err := io.Copy(squashLayerTarWriter, tf.Stream); err != nil {
						return err
					}
				}
				return nil
			}); err != nil {
				return err
			}
		case VERSION:
			uuidPart := nameParts[0]
			e.Layers[uuidPart].VersionHeader = t.Header
		}
		return nil<commit_msg>Oops, forgot to change these too<commit_after>package libsquash

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/winchman/libsquash/tarball"
)

var uuidRegex = regexp.MustCompile("^[a-f0-9]{64}$")

func (e *export) IngestImageMetadata(tarstream io.Reader) error {
	if err := tarball.Walk(tarstream, func(t *tarball.TarFile) error {
		switch Type(t) {
		case Ignore:
			return nil
		case Repositories:
			if err := json.NewDecoder(t.Stream).Decode(&e.Repositories); err != nil {
				return err
			}
			// Export may have multiple branches with the same parent. Abort.
			for _, v := range e.Repositories {
				commits := map[string]string{}
				for tag, commit := range *v {
					commits[commit] = tag
				}
				if len(commits) > 1 {
					return errorMultipleBranchesSameParent
				}
			}
		case JSON:
			uuid := t.NameParts()[0]
			if e.Layers[uuid] == nil {
				e.Layers[uuid] = &layer{}
			}
			e.Layers[uuid].JSONHeader = t.Header
			if err := json.NewDecoder(t.Stream).Decode(&e.Layers[uuid].LayerConfig); err != nil {
				return err
			}
		case LayerTar:
			uuid := t.NameParts()[0]
			if e.Layers[uuid] == nil {
				e.Layers[uuid] = &layer{}
			}
			if err := tarball.Walk(t.Stream, func(tf *tarball.TarFile) error {
				filePath := nameWithoutWhiteoutPrefix(tf.Name())
				if e.fileToLayers[filePath] == nil {
					e.fileToLayers[filePath] = []fileLoc{}
				}
				foundWhiteout := isWhiteout(tf.Name())
				e.fileToLayers[filePath] = append(e.fileToLayers[filePath], fileLoc{
					uuid:     uuid,
					whiteout: foundWhiteout,
				})

				if foundWhiteout {
					e.whiteouts = append(e.whiteouts, whiteoutFile{
						prefix: filePath,
						uuid:   uuid,
					})
				}
				return nil
			}); err != nil {
				return err
			}
		}
		return nil
	}); err != nil {
		return err
	}

	return e.populateFileData()
}

func (e *export) populateFileData() error {

	e.start = e.FirstSquash()

	// Can't find a previously squashed layer, default to root
	if e.start == nil {
		e.start = e.Root()
	}

	if e.start == nil {
		return errorNoFROM
	}

	index := 0
	current := e.start
	orderMap := map[string]int{}
	for {
		orderMap[current.LayerConfig.ID] = index
		index++
		current = e.ChildOf(current.LayerConfig.ID)
		if current == nil {
			break
		}
	}

	for path, fileLocs := range e.fileToLayers {
		if len(fileLocs) > 0 {
			greatest := fileLocs[0]
			for _, loc := range fileLocs {
				if orderMap[loc.uuid] > orderMap[greatest.uuid] {
					greatest = loc
				}
			}
			if e.layerToFiles[greatest.uuid] == nil {
				e.layerToFiles[greatest.uuid] = map[string]bool{}
			}

			// if name matches whiteout prefix and the whiteout file is found
			// in a layer that is >= greatest.uuid, skip
			uuidContainingWhiteout, matches := matchesWhiteout(path, e.whiteouts)
			if (matches && orderMap[uuidContainingWhiteout] >= orderMap[greatest.uuid]) || greatest.whiteout {
				delete(e.layerToFiles[greatest.uuid], path)
			} else {
				e.layerToFiles[greatest.uuid][path] = true
			}
		}
	}

	return nil
}

func (e *export) SquashLayers(into, from *layer, tarstream io.Reader, outstream io.Writer) (imageID string, err error) {
	tempfile, err := ioutil.TempFile("", "libsquash")
	if err != nil {
		return "", err
	}

	defer func() {
		_ = tempfile.Close()
		_ = os.RemoveAll(tempfile.Name())
	}()

	var squashLayerTarWriter = tar.NewWriter(tempfile)

	if err = tarball.Walk(tarstream, func(t *tarball.TarFile) error {
		nameParts := t.NameParts()
		switch Type(t) {
		case Directory:
			uuidPart := nameParts[0]
			e.Layers[uuidPart].DirHeader = t.Header
		case LayerTar:
			uuidPart := nameParts[0]
			e.Layers[uuidPart].LayerTarHeader = t.Header
			if err := tarball.Walk(t.Stream, func(tf *tarball.TarFile) error {
				filePath := nameWithoutWhiteoutPrefix(tf.Name())
				if e.layerToFiles[uuidPart][filePath] {
					squashLayerTarWriter.WriteHeader(tf.Header)
					if _, err := io.Copy(squashLayerTarWriter, tf.Stream); err != nil {
						return err
					}
				}
				return nil
			}); err != nil {
				return err
			}
		case Version:
			uuidPart := nameParts[0]
			e.Layers[uuidPart].VersionHeader = t.Header
		}
		return nil
{'fraction_non_alphanumeric': 0.13891885412921012, 'fraction_numerical': 0.003116384993407647, 'mean_word_length': 3.134786917740337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5346539', 'n_tokens_mistral': 3626, 'n_tokens_neox': 3015, 'n_words': 658}