<commit_before>// Package docker2aci implements a simple library for converting docker images to
// App Container Images (ACIs).
package docker2aci

import (
	"archive/tar"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/appc/spec/aci"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/docker/docker/runconfig"
)

type DockerImageData struct {
	ID            string            `json:"id"`
	Parent        string            `json:"parent,omitempty"`
	Comment       string            `json:"comment,omitempty"`
	Created       time.Time         `json:"created"`
	Container     string            `json:"container,omitempty"`
	DockerVersion string            `json:"docker_version,omitempty"`
	Author        string            `json:"author,omitempty"`
	Config        *runconfig.Config `json:"config,omitempty"`
	Architecture  string            `json:"architecture,omitempty"`
	OS            string            `json:"os,omitempty"`
	Checksum      string            `json:"checksum"`
}

type RepoData struct {
	Tokens    []string
	Endpoints []string
	Cookie    []string
}

type DockerURL struct {
	IndexURL  string
	ImageName string
	Tag       string
}

type SquashAccumulator struct {
	OutWriter *tar.Writer
	Manifests []schema.ImageManifest
	Filelist  []string
}

const (
	defaultIndex  = "index.docker.io"
	defaultTag    = "latest"
	schemaVersion = "0.1.1"
)

// Convert generates ACI images from docker registry URLs.
// It takes as input a dockerURL of the form:
//
// 	{docker registry URL}/{image name}:{tag}
//
// It then gets all the layers of the requested image and converts each of
// them to ACI.
// If the squash flag is true, it squashes all the layers in one file and
// places this file in outputDir; if it is false, it places every layer in its
// own ACI in outputDir.
// It returns the list of generated ACI paths.
func Convert(dockerURL string, squash bool, outputDir string) ([]string, error) {
	parsedURL := parseDockerURL(dockerURL)

	repoData, err := getRepoData(parsedURL.IndexURL, parsedURL.ImageName)
	if err != nil {
		return nil, fmt.Errorf("Error getting repository data: %v\n", err)
	}

	// TODO(iaguis) check more endpoints
	appImageID, err := getImageIDFromTag(repoData.Endpoints[0], parsedURL.ImageName, parsedURL.Tag, repoData)
	if err != nil {
		return nil, fmt.Errorf("Error getting ImageID from tag %s: %v\n", parsedURL.Tag, err)
	}

	ancestry, err := getAncestry(appImageID, repoData.Endpoints[0], repoData)
	if err != nil {
		return nil, fmt.Errorf("Error getting ancestry: %v\n", err)
	}

	layersOutputDir := "."
	if squash {
		layersOutputDir, err = ioutil.TempDir("", "docker2aci-")
		if err != nil {
			return nil, fmt.Errorf("Error creating dir: %v", err)
		}
		defer os.RemoveAll(layersOutputDir)
	}

	var aciLayerPaths []string
	for _, layerID := range ancestry {
		aciPath, err := buildACI(layerID, repoData, parsedURL, layersOutputDir)
		if err != nil {
			return nil, fmt.Errorf("Error building layer: %v\n", err)
		}

		aciLayerPaths = append(aciLayerPaths, aciPath)
	}

	if squash {
		squashedFilename := strings.Replace(parsedURL.ImageName, "/", "-", -1)
		if parsedURL.Tag != "" {
			squashedFilename += "-" + parsedURL.Tag
		}
		squashedFilename += ".aci"
		squashedImagePath := path.Join(outputDir, squashedFilename)

		if err := squashLayers(aciLayerPaths, squashedImagePath); err != nil {
			return nil, fmt.Errorf("Error squashing image: %v\n", err)
		}
		aciLayerPaths = []string{squashedImagePath}
	}

	return aciLayerPaths, nil
}

func parseDockerURL(arg string) *DockerURL {
	indexURL := defaultIndex
	tag := defaultTag

	argParts := strings.SplitN(arg, "/", 2)
	var appString string
	if len(argParts) > 1 {
		if strings.Contains(argParts[0], ".") {
			indexURL = argParts[0]
			appString = argParts[1]
		} else {
			appString = path.Join(argParts...)
		}
	} else {
		appString = argParts[0]
	}

	imageName := appString
	appParts := strings.Split(appString, ":")

	if len(appParts) > 1 {
		tag = appParts[len(appParts)-1]
		imageNameParts := appParts[0 : len(appParts)-1]
		imageName = strings.Join(imageNameParts, ":")
	}

	return &DockerURL{
		IndexURL:  indexURL,
		ImageName: imageName,
		Tag:       tag,
	}
}

func getRepoData(indexURL string, remote string) (*RepoData, error) {
	client := &http.Client{}
	repositoryURL := "https://" + path.Join(indexURL, "v1", "repositories", remote, "images")

	req, err := http.NewRequest("GET", repositoryURL, nil)
	if err != nil {
		return nil, err
	}

	// TODO(iaguis) add auth?
	req.Header.Set("X-Docker-Token", "true")

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()<commit_msg>Use docker library to parse the docker URL<commit_after>// Package docker2aci implements a simple library for converting docker images to
// App Container Images (ACIs).
package docker2aci

import (
	"archive/tar"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/appc/spec/aci"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/docker/docker/pkg/parsers"
	"github.com/docker/docker/registry"
	"github.com/docker/docker/runconfig"
)

type DockerImageData struct {
	ID            string            `json:"id"`
	Parent        string            `json:"parent,omitempty"`
	Comment       string            `json:"comment,omitempty"`
	Created       time.Time         `json:"created"`
	Container     string            `json:"container,omitempty"`
	DockerVersion string            `json:"docker_version,omitempty"`
	Author        string            `json:"author,omitempty"`
	Config        *runconfig.Config `json:"config,omitempty"`
	Architecture  string            `json:"architecture,omitempty"`
	OS            string            `json:"os,omitempty"`
	Checksum      string            `json:"checksum"`
}

type RepoData struct {
	Tokens    []string
	Endpoints []string
	Cookie    []string
}

type DockerURL struct {
	IndexURL  string
	ImageName string
	Tag       string
}

type SquashAccumulator struct {
	OutWriter *tar.Writer
	Manifests []schema.ImageManifest
	Filelist  []string
}

const (
	defaultIndex  = "index.docker.io"
	defaultTag    = "latest"
	schemaVersion = "0.1.1"
)

// Convert generates ACI images from docker registry URLs.
// It takes as input a dockerURL of the form:
//
// 	{docker registry URL}/{image name}:{tag}
//
// It then gets all the layers of the requested image and converts each of
// them to ACI.
// If the squash flag is true, it squashes all the layers in one file and
// places this file in outputDir; if it is false, it places every layer in its
// own ACI in outputDir.
// It returns the list of generated ACI paths.
func Convert(dockerURL string, squash bool, outputDir string) ([]string, error) {
	parsedURL, err := parseDockerURL(dockerURL)
	if err != nil {
		return nil, fmt.Errorf("error parsing docker url: %v\n", err)
	}

	repoData, err := getRepoData(parsedURL.IndexURL, parsedURL.ImageName)
	if err != nil {
		return nil, fmt.Errorf("Error getting repository data: %v\n", err)
	}

	// TODO(iaguis) check more endpoints
	appImageID, err := getImageIDFromTag(repoData.Endpoints[0], parsedURL.ImageName, parsedURL.Tag, repoData)
	if err != nil {
		return nil, fmt.Errorf("Error getting ImageID from tag %s: %v\n", parsedURL.Tag, err)
	}

	ancestry, err := getAncestry(appImageID, repoData.Endpoints[0], repoData)
	if err != nil {
		return nil, fmt.Errorf("Error getting ancestry: %v\n", err)
	}

	layersOutputDir := "."
	if squash {
		layersOutputDir, err = ioutil.TempDir("", "docker2aci-")
		if err != nil {
			return nil, fmt.Errorf("Error creating dir: %v", err)
		}
		defer os.RemoveAll(layersOutputDir)
	}

	var aciLayerPaths []string
	for _, layerID := range ancestry {
		aciPath, err := buildACI(layerID, repoData, parsedURL, layersOutputDir)
		if err != nil {
			return nil, fmt.Errorf("Error building layer: %v\n", err)
		}

		aciLayerPaths = append(aciLayerPaths, aciPath)
	}

	if squash {
		squashedFilename := strings.Replace(parsedURL.ImageName, "/", "-", -1)
		if parsedURL.Tag != "" {
			squashedFilename += "-" + parsedURL.Tag
		}
		squashedFilename += ".aci"
		squashedImagePath := path.Join(outputDir, squashedFilename)

		if err := squashLayers(aciLayerPaths, squashedImagePath); err != nil {
			return nil, fmt.Errorf("Error squashing image: %v\n", err)
		}
		aciLayerPaths = []string{squashedImagePath}
	}

	return aciLayerPaths, nil
}

func parseDockerURL(arg string) (*DockerURL, error) {
	taglessRemote, tag := parsers.ParseRepositoryTag(arg)
	if tag == "" {
		tag = defaultTag
	}
	repoInfo, err := registry.ParseRepositoryInfo(taglessRemote)
	if err != nil {
		return nil, err
	}
	indexURL := defaultIndex
	if !repoInfo.Index.Official {
		indexURL = repoInfo.Index.Name
	}

	return &DockerURL{
		IndexURL:  indexURL,
		ImageName: repoInfo.RemoteName,
		Tag:       tag,
	}, nil
}

func getRepoData(indexURL string, remote string) (*RepoData, error) {
	client := &http.Client{}
	repositoryURL := "https://" + path.Join(indexURL, "v1", "repositories", remote, "images")

	req, err := http.NewRequest("GET", repositoryURL, nil)
	if err != nil {
		return nil, err
	}

	// TODO(iaguis) add auth?
	req.Header.Set("X-Docker-Token", "true")

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
{'fraction_non_alphanumeric': 0.12922718117247958, 'fraction_numerical': 0.003180324393088095, 'mean_word_length': 3.4988078206962325, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6391216', 'n_tokens_mistral': 3307, 'n_tokens_neox': 3121, 'n_words': 847}