<commit_before>package kubernetes

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"time"

	"github.com/Azure/acs-engine/pkg/api/common"
	"github.com/Azure/acs-engine/test/e2e/config"
	"github.com/Azure/acs-engine/test/e2e/engine"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/deployment"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/job"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/networkpolicy"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/node"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/persistentvolumeclaims"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/pod"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/service"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/storageclass"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/util"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	WorkloadDir = "workloads"
)

var (
	cfg config.Config
	eng engine.Engine
)

var _ = BeforeSuite(func() {
	cwd, _ := os.Getwd()
	rootPath := filepath.Join(cwd, "../../..") // The current working dir of these tests is down a few levels from the root of the project. We should traverse up that path so we can find the _output dir
	c, err := config.ParseConfig()
	c.CurrentWorkingDir = rootPath
	Expect(err).NotTo(HaveOccurred())
	cfg = *c // We have to do this because golang anon functions and scoping and stuff

	engCfg, err := engine.ParseConfig(c.CurrentWorkingDir, c.ClusterDefinition, c.Name)
	Expect(err).NotTo(HaveOccurred())
	csInput, err := engine.ParseInput(engCfg.ClusterDefinitionTemplate)
	Expect(err).NotTo(HaveOccurred())
	csGenerated, err := engine.ParseOutput(engCfg.GeneratedDefinitionPath + "/apimodel.json")
	Expect(err).NotTo(HaveOccurred())
	eng = engine.Engine{
		Config:             engCfg,
		ClusterDefinition:  csInput,
		ExpandedDefinition: csGenerated,
	}
})

var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", func() {
	Describe("regardless of agent pool type", func() {
		It("should display the installed Ubuntu version on the master node", func() {
			kubeConfig, err := GetConfig()
			Expect(err).NotTo(HaveOccurred())
			master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
			sshKeyPath := cfg.GetSSHKeyPath()

			lsbReleaseCmd := fmt.Sprintf("lsb_release -a && uname -r")
			cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd)
			util.PrintCommand(cmd)
			out, err := cmd.CombinedOutput()
			log.Printf("%s\n", out)
			if err != nil {
				log.Printf("Error while getting Ubuntu image version: %s\n", out)
			}
		})

		It("should have have the appropriate node count", func() {
			nodeList, err := node.Get()
			Expect(err).NotTo(HaveOccurred())
			Expect(len(nodeList.Nodes)).To(Equal(eng.NodeCount()))
		})

		It("should have functional DNS", func() {
			if !eng.HasWindowsAgents() {
				if !eng.HasNetworkPolicy("calico") {
					var err error
					var p *pod.Pod
					p, err = pod.CreatePodFromFile(filepath.Join(WorkloadDir, "dns-liveness.yaml"), "dns-liveness", "default")
					if cfg.SoakClusterName == "" {
						Expect(err).NotTo(HaveOccurred())
					} else {
						if err != nil {
							p, err = pod.Get("dns-liveness", "default")
							Expect(err).NotTo(HaveOccurred())
						}
					}
					running, err := p.WaitOnReady(5*time.Second, 2*time.Minute)
					Expect(err).NotTo(HaveOccurred())
					Expect(running).To(Equal(true))
				}

				kubeConfig, err := GetConfig()
				Expect(err).NotTo(HaveOccurred())
				master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
				sshKeyPath := cfg.GetSSHKeyPath()

				ifconfigCmd := fmt.Sprintf("ifconfig -a -v")
				cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
				util.PrintCommand(cmd)
				out, err := cmd.CombinedOutput()
				log.Printf("%s\n", out)
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", out)
				}

				resolvCmd := fmt.Sprintf("cat /etc/resolv.conf")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				log.Printf("%s\n", out)
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", out)
				}

				By("Ensuring that we have a valid connection to our resolver")
				digCmd := fmt.Sprintf("dig +short +search +answer `hostname`")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", out)
				}

				nodeList, err := node.Get()
				Expect(err).NotTo(HaveOccurred())
				for _, node := range nodeList.Nodes {
					By("Ensuring that we get a DNS lookup answer response for each node hostname")
					digCmd := fmt.Sprintf("dig +short +search +answer %s | grep -v -e '^$'", node.Metadata.Name)
					cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
					util.PrintCommand(cmd)
					out, err = cmd.CombinedOutput()
					if err != nil {
						log.Printf("Error while querying DNS: %s\n", out)
					}
					Expect(err).NotTo(HaveOccurred())
				}

				By("Ensuring that we get a DNS lookup answer response for external names")
				digCmd = fmt.Sprintf("dig +short +search www.bing.com | grep -v -e '^$'")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", out)
				}
				digCmd = fmt.Sprintf("dig +short +search google.com | grep -v -e '^$'")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", out)
				}

				By("Ensuring that we get a DNS lookup answer response for external names using external resolver")
				digCmd = fmt.Sprintf("dig +short +search www.bing.com @8.8.8.8 | grep -v -e '^$'")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", out)
				}
				digCmd = fmt.Sprintf("dig +short +search google.com @8.8.8.8 | grep -v -e '^$'")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", out)
				}

				j, err := job.CreateJobFromFile(filepath.Join(WorkloadDir, "validate-dns.yaml"), "validate-dns", "default")
				Expect(err).NotTo(HaveOccurred())
				ready, err := j.WaitOnReady(5*time.Second, cfg.Timeout)
				delErr := j.Delete()
				if delErr != nil {
					fmt.Printf("could not delete job %s\n", j.Metadata.Name)
					fmt.Println(delErr)
				}
				Expect(err).NotTo(HaveOccurred())
				Expect(ready).To(Equal(true))
			}
		})

		It("should have kube-dns running", func() {
			running, err := pod.WaitOnReady("kube-dns", "kube-system", 3, 30*time.Second, cfg.Timeout)
			Expect(err).NotTo(HaveOccurred())<commit_msg>Kubernetes: print kernel version during E2E (#3633)<commit_after>package kubernetes

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"time"

	"github.com/Azure/acs-engine/pkg/api/common"
	"github.com/Azure/acs-engine/test/e2e/config"
	"github.com/Azure/acs-engine/test/e2e/engine"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/deployment"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/job"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/networkpolicy"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/node"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/persistentvolumeclaims"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/pod"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/service"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/storageclass"
	"github.com/Azure/acs-engine/test/e2e/kubernetes/util"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	WorkloadDir = "workloads"
)

var (
	cfg config.Config
	eng engine.Engine
)

var _ = BeforeSuite(func() {
	cwd, _ := os.Getwd()
	rootPath := filepath.Join(cwd, "../../..") // The current working dir of these tests is down a few levels from the root of the project. We should traverse up that path so we can find the _output dir
	c, err := config.ParseConfig()
	c.CurrentWorkingDir = rootPath
	Expect(err).NotTo(HaveOccurred())
	cfg = *c // We have to do this because golang anon functions and scoping and stuff

	engCfg, err := engine.ParseConfig(c.CurrentWorkingDir, c.ClusterDefinition, c.Name)
	Expect(err).NotTo(HaveOccurred())
	csInput, err := engine.ParseInput(engCfg.ClusterDefinitionTemplate)
	Expect(err).NotTo(HaveOccurred())
	csGenerated, err := engine.ParseOutput(engCfg.GeneratedDefinitionPath + "/apimodel.json")
	Expect(err).NotTo(HaveOccurred())
	eng = engine.Engine{
		Config:             engCfg,
		ClusterDefinition:  csInput,
		ExpandedDefinition: csGenerated,
	}
})

var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", func() {
	Describe("regardless of agent pool type", func() {
		It("should display the installed Ubuntu version on the master node", func() {
			kubeConfig, err := GetConfig()
			Expect(err).NotTo(HaveOccurred())
			master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
			sshKeyPath := cfg.GetSSHKeyPath()

			lsbReleaseCmd := fmt.Sprintf("lsb_release -a && uname -r")
			cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd)
			util.PrintCommand(cmd)
			out, err := cmd.CombinedOutput()
			log.Printf("%s\n", out)
			if err != nil {
				log.Printf("Error while getting Ubuntu image version: %s\n", err)
			}

			kernelVerCmd := fmt.Sprintf("cat /proc/version")
			cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, kernelVerCmd)
			util.PrintCommand(cmd)
			out, err = cmd.CombinedOutput()
			log.Printf("%s\n", out)
			if err != nil {
				log.Printf("Error while getting LinuxKernel version: %s\n", err)
			}
		})

		It("should have have the appropriate node count", func() {
			nodeList, err := node.Get()
			Expect(err).NotTo(HaveOccurred())
			Expect(len(nodeList.Nodes)).To(Equal(eng.NodeCount()))
		})

		It("should have functional DNS", func() {
			if !eng.HasWindowsAgents() {
				if !eng.HasNetworkPolicy("calico") {
					var err error
					var p *pod.Pod
					p, err = pod.CreatePodFromFile(filepath.Join(WorkloadDir, "dns-liveness.yaml"), "dns-liveness", "default")
					if cfg.SoakClusterName == "" {
						Expect(err).NotTo(HaveOccurred())
					} else {
						if err != nil {
							p, err = pod.Get("dns-liveness", "default")
							Expect(err).NotTo(HaveOccurred())
						}
					}
					running, err := p.WaitOnReady(5*time.Second, 2*time.Minute)
					Expect(err).NotTo(HaveOccurred())
					Expect(running).To(Equal(true))
				}

				kubeConfig, err := GetConfig()
				Expect(err).NotTo(HaveOccurred())
				master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
				sshKeyPath := cfg.GetSSHKeyPath()

				ifconfigCmd := fmt.Sprintf("ifconfig -a -v")
				cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
				util.PrintCommand(cmd)
				out, err := cmd.CombinedOutput()
				log.Printf("%s\n", out)
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", err)
				}

				resolvCmd := fmt.Sprintf("cat /etc/resolv.conf")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				log.Printf("%s\n", out)
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", err)
				}

				By("Ensuring that we have a valid connection to our resolver")
				digCmd := fmt.Sprintf("dig +short +search +answer `hostname`")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				log.Printf("%s\n", out)
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", err)
				}

				nodeList, err := node.Get()
				Expect(err).NotTo(HaveOccurred())
				for _, node := range nodeList.Nodes {
					By("Ensuring that we get a DNS lookup answer response for each node hostname")
					digCmd := fmt.Sprintf("dig +short +search +answer %s | grep -v -e '^$'", node.Metadata.Name)
					cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
					util.PrintCommand(cmd)
					out, err = cmd.CombinedOutput()
					log.Printf("%s\n", out)
					if err != nil {
						log.Printf("Error while querying DNS: %s\n", err)
					}
					Expect(err).NotTo(HaveOccurred())
				}

				By("Ensuring that we get a DNS lookup answer response for external names")
				digCmd = fmt.Sprintf("dig +short +search www.bing.com | grep -v -e '^$'")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", out)
				}
				digCmd = fmt.Sprintf("dig +short +search google.com | grep -v -e '^$'")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				log.Printf("%s\n", out)
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", err)
				}

				By("Ensuring that we get a DNS lookup answer response for external names using external resolver")
				digCmd = fmt.Sprintf("dig +short +search www.bing.com @8.8.8.8 | grep -v -e '^$'")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				log.Printf("%s\n", out)
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", err)
				}
				digCmd = fmt.Sprintf("dig +short +search google.com @8.8.8.8 | grep -v -e '^$'")
				cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
				util.PrintCommand(cmd)
				out, err = cmd.CombinedOutput()
				log.Printf("%s\n", out)
				if err != nil {
					log.Printf("Error while querying DNS: %s\n", err)
				}

				j, err := job.CreateJobFromFile(filepath.Join(WorkloadDir, "validate-dns.yaml"), "validate-dns", "default")
				Expect(err).NotTo(HaveOccurred())
				ready, err := j.WaitOnReady(5*time.Second, cfg.Timeout)
				delErr := j.Delete()
				if delErr != nil {
					fmt.Printf("could not delete job %s\n", j.Metadata.Name)
					fmt.Println(delErr)
				}
				Expect(err).NotTo(HaveOccurred())
				Expect(ready).To(Equal(true))
			}
		})

		It("should have kube-dns running", func() {
			running, err := pod.WaitOnReady("kube-dns", "kube-system", 3, 30*time.Second, cfg.Timeout)
			Expect(err).NotTo(HaveOccurred())
{'fraction_non_alphanumeric': 0.17229216079790793, 'fraction_numerical': 0.005655902207626346, 'mean_word_length': 3.7705250942848854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1544926', 'n_tokens_mistral': 6754, 'n_tokens_neox': 5761, 'n_words': 1250}