<commit_before>package cell_test

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"time"

	"github.com/pivotal-golang/archiver/extractor/test_helper"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
	"github.com/tedsuo/ifrit/grouper"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/inigo/helpers"
	"github.com/cloudfoundry-incubator/inigo/inigo_announcement_server"
	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/rep"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("Executor", func() {
	var (
		executorProcess, cellProcess ifrit.Process
	)

	var fileServerStaticDir string

	BeforeEach(func() {
		var fileServerRunner ifrit.Runner

		fileServerRunner, fileServerStaticDir = componentMaker.FileServer()
		executorProcess = ginkgomon.Invoke(componentMaker.Executor("-memoryMB", "1024"))

		cellGroup := grouper.Members{
			{"file-server", fileServerRunner},
			{"rep", componentMaker.Rep()},
			{"auctioneer", componentMaker.Auctioneer()},
			{"converger", componentMaker.Converger()},
		}
		cellProcess = ginkgomon.Invoke(grouper.NewParallel(os.Interrupt, cellGroup))
	})

	AfterEach(func() {
		helpers.StopProcesses(executorProcess, cellProcess)
	})

	Describe("Heartbeating", func() {
		It("should heartbeat its presence (through the rep)", func() {
			Eventually(receptorClient.Cells).Should(HaveLen(1))
		})
	})

	Describe("Resource limits", func() {
		It("should only pick up tasks if it has capacity", func() {
			firstGuyGuid := helpers.GenerateGuid()
			secondGuyGuid := helpers.GenerateGuid()

			err := receptorClient.CreateTask(helpers.TaskCreateRequestWithMemoryAndDisk(
				firstGuyGuid,
				&models.RunAction{
					Path: "/bin/bash",
					Args: []string{"-c", "curl " + inigo_announcement_server.AnnounceURL(firstGuyGuid) + " && tail -f /dev/null"},
				},
				1024,
				1024,
			))
			Ω(err).ShouldNot(HaveOccurred())

			Eventually(inigo_announcement_server.Announcements).Should(ContainElement(firstGuyGuid))

			err = receptorClient.CreateTask(helpers.TaskCreateRequestWithMemoryAndDisk(
				secondGuyGuid,
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(secondGuyGuid)},
				},
				1024,
				1024,
			))
			Ω(err).ShouldNot(HaveOccurred())

			Consistently(inigo_announcement_server.Announcements).ShouldNot(ContainElement(secondGuyGuid))
		})
	})

	Describe("consistency", func() {
		Context("when a task is running and then something causes the container to go away (e.g. executor restart)", func() {
			var taskGuid string

			BeforeEach(func() {
				taskGuid = helpers.GenerateGuid()

				err := receptorClient.CreateTask(helpers.TaskCreateRequest(
					taskGuid,
					&models.RunAction{
						Path: "sh",
						Args: []string{"-c", "while true; do sleep 1; done"},
					},
				))
				Ω(err).ShouldNot(HaveOccurred())

				executorClient := componentMaker.ExecutorClient()

				Eventually(func() executor.State {
					container, err := executorClient.GetContainer(taskGuid)
					if err == nil {
						return container.State
					}
					return executor.StateInvalid
				}).Should(Equal(executor.StateRunning))

				// bounce executor
				ginkgomon.Kill(executorProcess)
				executorProcess = ginkgomon.Invoke(componentMaker.Executor("-memoryMB", "1024"))
			})

			It("eventually marks the task completed and failed", func() {
				var task receptor.TaskResponse

				Eventually(func() interface{} {
					var err error

					task, err = receptorClient.GetTask(taskGuid)
					Ω(err).ShouldNot(HaveOccurred())

					return task.State
				}).Should(Equal(receptor.TaskStateCompleted))

				Ω(task.Failed).Should(BeTrue())
			})
		})

		Context("when a lrp is running and then something causes the container to go away", func() {
			var (
				instanceGuid string
				processGuid  string
				index        int
			)

			BeforeEach(func() {
				processGuid = helpers.GenerateGuid()
				index = 0

				err := receptorClient.CreateDesiredLRP(helpers.LightweightLRPCreateRequest(processGuid))
				Ω(err).ShouldNot(HaveOccurred())

				var actualLRPs []receptor.ActualLRPResponse
				Eventually(func() interface{} {
					actualLRPs = helpers.ActiveActualLRPs(receptorClient, processGuid)
					return actualLRPs
				}).Should(HaveLen(1))

				instanceGuid = actualLRPs[0].InstanceGuid
				containerGuid := rep.LRPContainerGuid(processGuid, instanceGuid)

				executorClient := componentMaker.ExecutorClient()

				Eventually(func() executor.State {
					container, err := executorClient.GetContainer(containerGuid)
					if err == nil {
						return container.State
					}

					return executor.StateInvalid
				}).Should(Equal(executor.StateRunning))

				// bounce executor
				ginkgomon.Kill(executorProcess)
				executorProcess = ginkgomon.Invoke(componentMaker.Executor("-memoryMB", "1024"))
			})

			It("eventually deletes the original lrp", func() {
				lrpMatchesOriginalInstanceGuid := func() (bool, error) {
					actualLRP, err := receptorClient.ActualLRPByProcessGuidAndIndex(processGuid, index)
					if err != nil {
						return false, err
					}
					return actualLRP.InstanceGuid == instanceGuid, nil
				}

				Eventually(lrpMatchesOriginalInstanceGuid).Should(BeFalse())
			})
		})
	})

	Describe("Preloaded RootFSes", func() {
		It("should only pick up tasks if the preloaded rootfses match", func() {
			matchingGuid := helpers.GenerateGuid()
			matchingGuid2 := helpers.GenerateGuid()
			nonMatchingGuid := helpers.GenerateGuid()

			err := receptorClient.CreateTask(helpers.TaskCreateRequest(
				matchingGuid,
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(matchingGuid)},
				},
			))
			Ω(err).ShouldNot(HaveOccurred())

			err = receptorClient.CreateTask(helpers.TaskCreateRequestWithRootFS(
				matchingGuid2,
				helpers.SecondaryPreloadedRootFS,
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(matchingGuid2)},
				},
			))
			Ω(err).ShouldNot(HaveOccurred())

			err = receptorClient.CreateTask(helpers.TaskCreateRequestWithRootFS(
				nonMatchingGuid,
				helpers.BogusPreloadedRootFS,
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(nonMatchingGuid)},
				},
			))
			Ω(err).ShouldNot(HaveOccurred())

			Consistently(inigo_announcement_server.Announcements).ShouldNot(ContainElement(nonMatchingGuid), "Did not expect to see this app running, as it has the wrong stack.")
			Eventually(inigo_announcement_server.Announcements).Should(ContainElement(matchingGuid))
			Eventually(inigo_announcement_server.Announcements).Should(ContainElement(matchingGuid2))
		})
	})

	Describe("'Arbitrary' RootFSes", func() {
		It("should only pick up tasks if the arbitrary rootfses match", func() {
			matchingGuid := helpers.GenerateGuid()
			nonMatchingGuid := helpers.GenerateGuid()

			err := receptorClient.CreateTask(helpers.TaskCreateRequestWithRootFS(
				matchingGuid,
				"docker:///concourse/busyboxplus#curl",
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(matchingGuid)},
				},
			))
			Ω(err).ShouldNot(HaveOccurred())
<commit_msg>Use cloudfoundry docker image, not concourse [#90425196]<commit_after>package cell_test

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"time"

	"github.com/pivotal-golang/archiver/extractor/test_helper"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
	"github.com/tedsuo/ifrit/grouper"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/inigo/helpers"
	"github.com/cloudfoundry-incubator/inigo/inigo_announcement_server"
	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/rep"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("Executor", func() {
	var (
		executorProcess, cellProcess ifrit.Process
	)

	var fileServerStaticDir string

	BeforeEach(func() {
		var fileServerRunner ifrit.Runner

		fileServerRunner, fileServerStaticDir = componentMaker.FileServer()
		executorProcess = ginkgomon.Invoke(componentMaker.Executor("-memoryMB", "1024"))

		cellGroup := grouper.Members{
			{"file-server", fileServerRunner},
			{"rep", componentMaker.Rep()},
			{"auctioneer", componentMaker.Auctioneer()},
			{"converger", componentMaker.Converger()},
		}
		cellProcess = ginkgomon.Invoke(grouper.NewParallel(os.Interrupt, cellGroup))
	})

	AfterEach(func() {
		helpers.StopProcesses(executorProcess, cellProcess)
	})

	Describe("Heartbeating", func() {
		It("should heartbeat its presence (through the rep)", func() {
			Eventually(receptorClient.Cells).Should(HaveLen(1))
		})
	})

	Describe("Resource limits", func() {
		It("should only pick up tasks if it has capacity", func() {
			firstGuyGuid := helpers.GenerateGuid()
			secondGuyGuid := helpers.GenerateGuid()

			err := receptorClient.CreateTask(helpers.TaskCreateRequestWithMemoryAndDisk(
				firstGuyGuid,
				&models.RunAction{
					Path: "/bin/bash",
					Args: []string{"-c", "curl " + inigo_announcement_server.AnnounceURL(firstGuyGuid) + " && tail -f /dev/null"},
				},
				1024,
				1024,
			))
			Ω(err).ShouldNot(HaveOccurred())

			Eventually(inigo_announcement_server.Announcements).Should(ContainElement(firstGuyGuid))

			err = receptorClient.CreateTask(helpers.TaskCreateRequestWithMemoryAndDisk(
				secondGuyGuid,
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(secondGuyGuid)},
				},
				1024,
				1024,
			))
			Ω(err).ShouldNot(HaveOccurred())

			Consistently(inigo_announcement_server.Announcements).ShouldNot(ContainElement(secondGuyGuid))
		})
	})

	Describe("consistency", func() {
		Context("when a task is running and then something causes the container to go away (e.g. executor restart)", func() {
			var taskGuid string

			BeforeEach(func() {
				taskGuid = helpers.GenerateGuid()

				err := receptorClient.CreateTask(helpers.TaskCreateRequest(
					taskGuid,
					&models.RunAction{
						Path: "sh",
						Args: []string{"-c", "while true; do sleep 1; done"},
					},
				))
				Ω(err).ShouldNot(HaveOccurred())

				executorClient := componentMaker.ExecutorClient()

				Eventually(func() executor.State {
					container, err := executorClient.GetContainer(taskGuid)
					if err == nil {
						return container.State
					}
					return executor.StateInvalid
				}).Should(Equal(executor.StateRunning))

				// bounce executor
				ginkgomon.Kill(executorProcess)
				executorProcess = ginkgomon.Invoke(componentMaker.Executor("-memoryMB", "1024"))
			})

			It("eventually marks the task completed and failed", func() {
				var task receptor.TaskResponse

				Eventually(func() interface{} {
					var err error

					task, err = receptorClient.GetTask(taskGuid)
					Ω(err).ShouldNot(HaveOccurred())

					return task.State
				}).Should(Equal(receptor.TaskStateCompleted))

				Ω(task.Failed).Should(BeTrue())
			})
		})

		Context("when a lrp is running and then something causes the container to go away", func() {
			var (
				instanceGuid string
				processGuid  string
				index        int
			)

			BeforeEach(func() {
				processGuid = helpers.GenerateGuid()
				index = 0

				err := receptorClient.CreateDesiredLRP(helpers.LightweightLRPCreateRequest(processGuid))
				Ω(err).ShouldNot(HaveOccurred())

				var actualLRPs []receptor.ActualLRPResponse
				Eventually(func() interface{} {
					actualLRPs = helpers.ActiveActualLRPs(receptorClient, processGuid)
					return actualLRPs
				}).Should(HaveLen(1))

				instanceGuid = actualLRPs[0].InstanceGuid
				containerGuid := rep.LRPContainerGuid(processGuid, instanceGuid)

				executorClient := componentMaker.ExecutorClient()

				Eventually(func() executor.State {
					container, err := executorClient.GetContainer(containerGuid)
					if err == nil {
						return container.State
					}

					return executor.StateInvalid
				}).Should(Equal(executor.StateRunning))

				// bounce executor
				ginkgomon.Kill(executorProcess)
				executorProcess = ginkgomon.Invoke(componentMaker.Executor("-memoryMB", "1024"))
			})

			It("eventually deletes the original lrp", func() {
				lrpMatchesOriginalInstanceGuid := func() (bool, error) {
					actualLRP, err := receptorClient.ActualLRPByProcessGuidAndIndex(processGuid, index)
					if err != nil {
						return false, err
					}
					return actualLRP.InstanceGuid == instanceGuid, nil
				}

				Eventually(lrpMatchesOriginalInstanceGuid).Should(BeFalse())
			})
		})
	})

	Describe("Preloaded RootFSes", func() {
		It("should only pick up tasks if the preloaded rootfses match", func() {
			matchingGuid := helpers.GenerateGuid()
			matchingGuid2 := helpers.GenerateGuid()
			nonMatchingGuid := helpers.GenerateGuid()

			err := receptorClient.CreateTask(helpers.TaskCreateRequest(
				matchingGuid,
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(matchingGuid)},
				},
			))
			Ω(err).ShouldNot(HaveOccurred())

			err = receptorClient.CreateTask(helpers.TaskCreateRequestWithRootFS(
				matchingGuid2,
				helpers.SecondaryPreloadedRootFS,
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(matchingGuid2)},
				},
			))
			Ω(err).ShouldNot(HaveOccurred())

			err = receptorClient.CreateTask(helpers.TaskCreateRequestWithRootFS(
				nonMatchingGuid,
				helpers.BogusPreloadedRootFS,
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(nonMatchingGuid)},
				},
			))
			Ω(err).ShouldNot(HaveOccurred())

			Consistently(inigo_announcement_server.Announcements).ShouldNot(ContainElement(nonMatchingGuid), "Did not expect to see this app running, as it has the wrong stack.")
			Eventually(inigo_announcement_server.Announcements).Should(ContainElement(matchingGuid))
			Eventually(inigo_announcement_server.Announcements).Should(ContainElement(matchingGuid2))
		})
	})

	Describe("'Arbitrary' RootFSes", func() {
		It("should only pick up tasks if the arbitrary rootfses match", func() {
			matchingGuid := helpers.GenerateGuid()
			nonMatchingGuid := helpers.GenerateGuid()

			err := receptorClient.CreateTask(helpers.TaskCreateRequestWithRootFS(
				matchingGuid,
				"docker:///cloudfoundry/busyboxplus#curl",
				&models.RunAction{
					Path: "curl",
					Args: []string{inigo_announcement_server.AnnounceURL(matchingGuid)},
				},
			))
			Ω(err).ShouldNot(HaveOccurred())

{'fraction_non_alphanumeric': 0.12435793457691267, 'fraction_numerical': 0.005542038388753718, 'mean_word_length': 4.290311047550947, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '619823', 'n_tokens_mistral': 5832, 'n_tokens_neox': 4709, 'n_words': 782}