<commit_before>package repository_fetcher_test

import (
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/docker/docker/archive"
	"github.com/docker/docker/image"
	"github.com/docker/docker/registry"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/cloudfoundry-incubator/warden-linux/linux_backend/container_pool/fake_graph"
	. "github.com/cloudfoundry-incubator/warden-linux/linux_backend/container_pool/repository_fetcher"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("RepositoryFetcher", func() {
	var graph *fake_graph.FakeGraph
	var fetcher RepositoryFetcher

	var logger *lagertest.TestLogger

	var server *ghttp.Server
	var endpoint1 *ghttp.Server
	var endpoint2 *ghttp.Server

	BeforeEach(func() {
		graph = fake_graph.New()

		server = ghttp.NewServer()

		endpoint1 = ghttp.NewServer()
		endpoint2 = ghttp.NewServer()

		registry, err := registry.NewRegistry(nil, nil, server.URL()+"/v1/", true)
		Ω(err).ShouldNot(HaveOccurred())

		fetcher = New(registry, graph)

		logger = lagertest.NewTestLogger("test")
	})

	setupSuccessfulFetch := func(endpoint *ghttp.Server) {
		endpoint.AppendHandlers(
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-3/json"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Header().Add("X-Docker-Size", "123")
					w.Write([]byte(`{"id":"layer-3","parent":"parent-3","Config":{"env": ["env2=env2Value"]}}`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-3/layer"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Write([]byte(`layer-3-data`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-2/json"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Header().Add("X-Docker-Size", "456")
					w.Write([]byte(`{"id":"layer-2","parent":"parent-2","Config":{"env": ["env1=env1Value", "env2=env2BadValue"]}}`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-2/layer"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Write([]byte(`layer-2-data`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-1/json"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Header().Add("X-Docker-Size", "789")
					w.Write([]byte(`{"id":"layer-1","parent":"parent-1"}`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-1/layer"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Write([]byte(`layer-1-data`))
				}),
			),
		)
	}

	Describe("Fetch", func() {
		BeforeEach(func() {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/v1/repositories/some-repo/images"),
					http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
						w.Header().Set("X-Docker-Token", "token-1,token-2")
						w.Header().Add("X-Docker-Endpoints", endpoint1.HTTPTestServer.Listener.Addr().String())
						w.Header().Add("X-Docker-Endpoints", endpoint2.HTTPTestServer.Listener.Addr().String())
						w.Write([]byte(`[
							{"id": "id-1", "checksum": "sha-1"},
							{"id": "id-2", "checksum": "sha-2"}
						]`))
					}),
				),
			)

			endpoint1.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/v1/repositories/library/some-repo/tags"),
					http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
						w.Write([]byte(`{
							"some-tag": "id-1",
							"some-other-tag": "id-2"
						}`))
					}),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/v1/images/id-1/ancestry"),
					http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
						w.Write([]byte(`["layer-1", "layer-2", "layer-3"]`))
					}),
				),
			)
		})

		Context("when none of the layers already exist", func() {
			BeforeEach(func() {
				setupSuccessfulFetch(endpoint1)
			})

			It("downloads all layers of the given tag of a repository and returns its image id", func() {
				expectedLayerNum := 3

				graph.WhenRegistering = func(imageJSON []byte, layer archive.ArchiveReader, image *image.Image) error {
					if expectedLayerNum == 3 {
						Ω(string(imageJSON)).Should(Equal(fmt.Sprintf(
							`{"id":"layer-%d","parent":"parent-%d","Config":{"env": ["env2=env2Value"]}}`,
							expectedLayerNum,
							expectedLayerNum,
						)))
					} else if expectedLayerNum == 2 {
						Ω(string(imageJSON)).Should(Equal(fmt.Sprintf(
							`{"id":"layer-%d","parent":"parent-%d","Config":{"env": ["env1=env1Value", "env2=env2BadValue"]}}`,
							expectedLayerNum,
							expectedLayerNum,
						)))
					} else {
						Ω(string(imageJSON)).Should(Equal(fmt.Sprintf(
							`{"id":"layer-%d","parent":"parent-%d"}`,
							expectedLayerNum,
							expectedLayerNum,
						)))
					}

					Ω(image.ID).Should(Equal(fmt.Sprintf("layer-%d", expectedLayerNum)))
					Ω(image.Parent).Should(Equal(fmt.Sprintf("parent-%d", expectedLayerNum)))

					layerData, err := ioutil.ReadAll(layer)
					Ω(err).ShouldNot(HaveOccurred())
					Ω(string(layerData)).Should(Equal(fmt.Sprintf("layer-%d-data", expectedLayerNum)))

					expectedLayerNum--

					return nil
				}

				imageID, envvars, err := fetcher.Fetch(logger, "some-repo", "some-tag")

				Ω(err).ShouldNot(HaveOccurred())
				Ω(envvars).Should(Equal([]string{"env2=env2Value", "env1=env1Value"}))
				Ω(imageID).Should(Equal("id-1"))
			})

			Context("when the first endpoint fails", func() {
				BeforeEach(func() {
					endpoint1.SetHandler(1, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {<commit_msg>Fixed test case to no longer depends on HashMap order [#75752312]<commit_after>package repository_fetcher_test

import (
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/docker/docker/archive"
	"github.com/docker/docker/image"
	"github.com/docker/docker/registry"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/cloudfoundry-incubator/warden-linux/linux_backend/container_pool/fake_graph"
	. "github.com/cloudfoundry-incubator/warden-linux/linux_backend/container_pool/repository_fetcher"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("RepositoryFetcher", func() {
	var graph *fake_graph.FakeGraph
	var fetcher RepositoryFetcher

	var logger *lagertest.TestLogger

	var server *ghttp.Server
	var endpoint1 *ghttp.Server
	var endpoint2 *ghttp.Server

	BeforeEach(func() {
		graph = fake_graph.New()

		server = ghttp.NewServer()

		endpoint1 = ghttp.NewServer()
		endpoint2 = ghttp.NewServer()

		registry, err := registry.NewRegistry(nil, nil, server.URL()+"/v1/", true)
		Ω(err).ShouldNot(HaveOccurred())

		fetcher = New(registry, graph)

		logger = lagertest.NewTestLogger("test")
	})

	setupSuccessfulFetch := func(endpoint *ghttp.Server) {
		endpoint.AppendHandlers(
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-3/json"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Header().Add("X-Docker-Size", "123")
					w.Write([]byte(`{"id":"layer-3","parent":"parent-3","Config":{"env": ["env2=env2Value"]}}`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-3/layer"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Write([]byte(`layer-3-data`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-2/json"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Header().Add("X-Docker-Size", "456")
					w.Write([]byte(`{"id":"layer-2","parent":"parent-2","Config":{"env": ["env1=env1Value", "env2=env2BadValue"]}}`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-2/layer"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Write([]byte(`layer-2-data`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-1/json"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Header().Add("X-Docker-Size", "789")
					w.Write([]byte(`{"id":"layer-1","parent":"parent-1"}`))
				}),
			),
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", "/v1/images/layer-1/layer"),
				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					w.Write([]byte(`layer-1-data`))
				}),
			),
		)
	}

	Describe("Fetch", func() {
		BeforeEach(func() {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/v1/repositories/some-repo/images"),
					http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
						w.Header().Set("X-Docker-Token", "token-1,token-2")
						w.Header().Add("X-Docker-Endpoints", endpoint1.HTTPTestServer.Listener.Addr().String())
						w.Header().Add("X-Docker-Endpoints", endpoint2.HTTPTestServer.Listener.Addr().String())
						w.Write([]byte(`[
							{"id": "id-1", "checksum": "sha-1"},
							{"id": "id-2", "checksum": "sha-2"}
						]`))
					}),
				),
			)

			endpoint1.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/v1/repositories/library/some-repo/tags"),
					http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
						w.Write([]byte(`{
							"some-tag": "id-1",
							"some-other-tag": "id-2"
						}`))
					}),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/v1/images/id-1/ancestry"),
					http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
						w.Write([]byte(`["layer-1", "layer-2", "layer-3"]`))
					}),
				),
			)
		})

		Context("when none of the layers already exist", func() {
			BeforeEach(func() {
				setupSuccessfulFetch(endpoint1)
			})

			It("downloads all layers of the given tag of a repository and returns its image id", func() {
				expectedLayerNum := 3

				graph.WhenRegistering = func(imageJSON []byte, layer archive.ArchiveReader, image *image.Image) error {
					if expectedLayerNum == 3 {
						Ω(string(imageJSON)).Should(Equal(fmt.Sprintf(
							`{"id":"layer-%d","parent":"parent-%d","Config":{"env": ["env2=env2Value"]}}`,
							expectedLayerNum,
							expectedLayerNum,
						)))
					} else if expectedLayerNum == 2 {
						Ω(string(imageJSON)).Should(Equal(fmt.Sprintf(
							`{"id":"layer-%d","parent":"parent-%d","Config":{"env": ["env1=env1Value", "env2=env2BadValue"]}}`,
							expectedLayerNum,
							expectedLayerNum,
						)))
					} else {
						Ω(string(imageJSON)).Should(Equal(fmt.Sprintf(
							`{"id":"layer-%d","parent":"parent-%d"}`,
							expectedLayerNum,
							expectedLayerNum,
						)))
					}

					Ω(image.ID).Should(Equal(fmt.Sprintf("layer-%d", expectedLayerNum)))
					Ω(image.Parent).Should(Equal(fmt.Sprintf("parent-%d", expectedLayerNum)))

					layerData, err := ioutil.ReadAll(layer)
					Ω(err).ShouldNot(HaveOccurred())
					Ω(string(layerData)).Should(Equal(fmt.Sprintf("layer-%d-data", expectedLayerNum)))

					expectedLayerNum--

					return nil
				}

				imageID, envvars, err := fetcher.Fetch(logger, "some-repo", "some-tag")

				Ω(err).ShouldNot(HaveOccurred())
				Ω(envvars).Should(ContainElement("env2=env2Value"))
				Ω(envvars).Should(ContainElement("env1=env1Value"))
				Ω(imageID).Should(Equal("id-1"))
			})

			Context("when the first endpoint fails", func() {
				BeforeEach(func() {
					endpoint1.SetHandler(1, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
{'fraction_non_alphanumeric': 0.18962386511024643, 'fraction_numerical': 0.013834846519671422, 'mean_word_length': 3.783291976840364, 'pattern_counts': {'":': 52, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '984462', 'n_tokens_mistral': 4950, 'n_tokens_neox': 4048, 'n_words': 532}