<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/client"
)

// This uses Docker to convert a Docker image into a tarball. It would be an improvement if we
// used the containerd libraries to do this instead locally direct from a local image
// cache as it would be much simpler.

var exclude = map[string]bool{
	".dockerenv":  true,
	"Dockerfile":  true,
	"dev/console": true,
	"dev/pts":     true,
	"dev/shm":     true,
}

var replace = map[string]string{
	"etc/hosts": `127.0.0.1       localhost
::1     localhost ip6-localhost ip6-loopback
fc00:db20:35b:7399::5 ip6-localnet
fc00:e968:6179::de52:7100 ip6-mcastprefix
fc00:e968:6179::de52:7100 ip6-allnodes
fc00:e968:6179::de52:7100 ip6-allrouters
`,
	"etc/resolv.conf": `nameserver 8.8.8.8
nameserver 8.8.4.4
nameserver fc00:db20:35b:7399::5
nameserver fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b
`,
	"etc/hostname": "moby",
}

// ImageExtract extracts the filesystem from an image and returns a tarball with the files prefixed by the given path
func ImageExtract(image, prefix string, trust bool, pull bool) ([]byte, error) {
	log.Debugf("image extract: %s %s", image, prefix)
	out := new(bytes.Buffer)
	tw := tar.NewWriter(out)
	err := tarPrefix(prefix, tw)
	if err != nil {
		return []byte{}, err
	}
	err = imageTar(image, prefix, tw, trust, pull)
	if err != nil {
		return []byte{}, err
	}
	err = tw.Close()
	if err != nil {
		return []byte{}, err
	}
	return out.Bytes(), nil
}

// tarPrefix creates the leading directories for a path
func tarPrefix(path string, tw *tar.Writer) error {
	if path == "" {
		return nil
	}
	if path[len(path)-1] != byte('/') {
		return fmt.Errorf("path does not end with /: %s", path)
	}
	path = path[:len(path)-1]
	if path[0] == byte('/') {
		return fmt.Errorf("path should be relative: %s", path)
	}
	mkdir := ""
	for _, dir := range strings.Split(path, "/") {
		mkdir = mkdir + dir
		hdr := &tar.Header{
			Name:     mkdir,
			Mode:     0755,
			Typeflag: tar.TypeDir,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		mkdir = mkdir + "/"
	}
	return nil
}

func imageTar(image, prefix string, tw *tar.Writer, trust bool, pull bool) error {
	log.Debugf("image tar: %s %s", image, prefix)
	if prefix != "" && prefix[len(prefix)-1] != byte('/') {
		return fmt.Errorf("prefix does not end with /: %s", prefix)
	}

	if pull || trust {
		log.Infof("Pull image: %s", image)
		err := dockerPull(image, trust)
		if err != nil {
			return fmt.Errorf("Could not pull image %s: %v", image, err)
		}
	}
	container, err := dockerCreate(image)
	if err != nil {
		// if the image wasn't found, pull it down.  Bail on other errors.
		if client.IsErrNotFound(err) {
			log.Infof("Pull image: %s", image)
			err := dockerPull(image, trust)
			if err != nil {
				return fmt.Errorf("Could not pull image %s: %v", image, err)
			}
			container, err = dockerCreate(image)
			if err != nil {
				return fmt.Errorf("Failed to docker create image %s: %v", image, err)
			}
		}
		return fmt.Errorf("Failed to create docker image %s: %v", image, err)

	}
	contents, err := dockerExport(container)
	if err != nil {
		return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
	}
	err = dockerRm(container)
	if err != nil {<commit_msg>Don't exit after pulling init or kernel image<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/client"
)

// This uses Docker to convert a Docker image into a tarball. It would be an improvement if we
// used the containerd libraries to do this instead locally direct from a local image
// cache as it would be much simpler.

var exclude = map[string]bool{
	".dockerenv":  true,
	"Dockerfile":  true,
	"dev/console": true,
	"dev/pts":     true,
	"dev/shm":     true,
}

var replace = map[string]string{
	"etc/hosts": `127.0.0.1       localhost
::1     localhost ip6-localhost ip6-loopback
fc00:db20:35b:7399::5 ip6-localnet
fc00:e968:6179::de52:7100 ip6-mcastprefix
fc00:e968:6179::de52:7100 ip6-allnodes
fc00:e968:6179::de52:7100 ip6-allrouters
`,
	"etc/resolv.conf": `nameserver 8.8.8.8
nameserver 8.8.4.4
nameserver fc00:db20:35b:7399::5
nameserver fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b
`,
	"etc/hostname": "moby",
}

// ImageExtract extracts the filesystem from an image and returns a tarball with the files prefixed by the given path
func ImageExtract(image, prefix string, trust bool, pull bool) ([]byte, error) {
	log.Debugf("image extract: %s %s", image, prefix)
	out := new(bytes.Buffer)
	tw := tar.NewWriter(out)
	err := tarPrefix(prefix, tw)
	if err != nil {
		return []byte{}, err
	}
	err = imageTar(image, prefix, tw, trust, pull)
	if err != nil {
		return []byte{}, err
	}
	err = tw.Close()
	if err != nil {
		return []byte{}, err
	}
	return out.Bytes(), nil
}

// tarPrefix creates the leading directories for a path
func tarPrefix(path string, tw *tar.Writer) error {
	if path == "" {
		return nil
	}
	if path[len(path)-1] != byte('/') {
		return fmt.Errorf("path does not end with /: %s", path)
	}
	path = path[:len(path)-1]
	if path[0] == byte('/') {
		return fmt.Errorf("path should be relative: %s", path)
	}
	mkdir := ""
	for _, dir := range strings.Split(path, "/") {
		mkdir = mkdir + dir
		hdr := &tar.Header{
			Name:     mkdir,
			Mode:     0755,
			Typeflag: tar.TypeDir,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		mkdir = mkdir + "/"
	}
	return nil
}

func imageTar(image, prefix string, tw *tar.Writer, trust bool, pull bool) error {
	log.Debugf("image tar: %s %s", image, prefix)
	if prefix != "" && prefix[len(prefix)-1] != byte('/') {
		return fmt.Errorf("prefix does not end with /: %s", prefix)
	}

	if pull || trust {
		log.Infof("Pull image: %s", image)
		err := dockerPull(image, trust)
		if err != nil {
			return fmt.Errorf("Could not pull image %s: %v", image, err)
		}
	}
	container, err := dockerCreate(image)
	if err != nil {
		// if the image wasn't found, pull it down.  Bail on other errors.
		if client.IsErrNotFound(err) {
			log.Infof("Pull image: %s", image)
			err := dockerPull(image, trust)
			if err != nil {
				return fmt.Errorf("Could not pull image %s: %v", image, err)
			}
			container, err = dockerCreate(image)
			if err != nil {
				return fmt.Errorf("Failed to docker create image %s: %v", image, err)
			}
		} else {
			return fmt.Errorf("Failed to create docker image %s: %v", image, err)
		}
	}
	contents, err := dockerExport(container)
	if err != nil {
		return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
	}
	err = dockerRm(container)
	if err != nil {
{'fraction_non_alphanumeric': 0.15224145583666224, 'fraction_numerical': 0.0340287024707797, 'mean_word_length': 3.5955132562882395, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3662036', 'n_tokens_mistral': 2734, 'n_tokens_neox': 2479, 'n_words': 750}