func (c *testContext) makeSSHCallWithRetries(ip, port, cmd string, retryCount int) {
	for ; retryCount > 0; retryCount-- {
		out, eout, err := c.sshCall(ip, port, cmd)
		if err == nil {
			return
		}
		log.Infof("Call '%s' failed: %s, %s, %v", cmd, out, eout, err)
		time.Sleep(30 * time.Second)
	}
	require.FailNow(c.t, fmt.Sprintf("SSH call failed all retries"))
}
Make an ssh call and fail if it errors after "n" retries; display any errors that occur
package test
import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"
	log "github.com/sirupsen/logrus"
	"github.com/stretchr/testify/require"
	existinginfrav1 "github.com/weaveworks/cluster-api-provider-existinginfra/apis/cluster.weave.works/v1alpha3"
	capeios "github.com/weaveworks/cluster-api-provider-existinginfra/pkg/apis/wksprovider/machine/os"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan/recipe"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan/resource"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan/runners/ssh"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan/runners/sudo"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/specs"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/utilities/envcfg"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/utilities/object"
)
type testContext struct {
	t       *testing.T
	tmpDir  string
	testDir string
}
type commandConfig struct {
	CheckError bool
	Dir        string
	Env        []string
	Stdout     *os.File
	Stderr     *os.File
}
func getTestContext(t *testing.T) *testContext {
	tmpDir, err := ioutil.TempDir("", "tmp_dir")
	require.NoError(t, err)
	return getTestContextFrom(t, tmpDir)
}
func getTestContextFrom(t *testing.T, tmpDir string) *testContext {
	log.Infof("Using temporary directory: %s\n", tmpDir)
	c := &testContext{
		t:       t,
		tmpDir:  tmpDir,
		testDir: getTestDir(t),
	}
	log.Info("Installing kind...")
	c.runWithConfig(commandConfig{CheckError: true, Env: env("GO111MODULE=on", "HOME="+tmpDir, "CGO_ENABLED=0"), Stdout: os.Stdout, Stderr: os.Stderr},
		"go", "get", "sigs.k8s.io/kind@v0.9.0")
	log.Info("Installing footloose...")
	c.runWithConfig(commandConfig{CheckError: true, Env: env("GO111MODULE=on", "HOME="+tmpDir, "CGO_ENABLED=0"), Stdout: os.Stdout, Stderr: os.Stderr},
		"go", "get", "github.com/weaveworks/footloose")
	log.Info("Installing clusterctl...")
	for {
		fetchCmd := fmt.Sprintf("curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.10/clusterctl-%s-%s -o %s/clusterctl && chmod a+x %s/clusterctl",
			c.getOS(), c.getArch(), c.tmpDir, c.tmpDir)
		c.runAndCheckError("sh", "-c", fetchCmd)
		_, _, err := c.runCollectingOutput(filepath.Join(c.tmpDir, "clusterctl"), "version")
		if err == nil {
			break
		}
	}
	err := os.Setenv("HOME", c.tmpDir)
	require.NoError(c.t, err)
	kindCmd := filepath.Join(c.tmpDir, "go", "bin", "kind")
	c.run(kindCmd, "delete", "cluster")
	c.runAndCheckError(kindCmd, "create", "cluster")
	return c
}
func (c *testContext) cleanup() {
	log.Infof("About to remove temp dir: '%s'", c.tmpDir)
	c.runAndCheckError(filepath.Join(c.tmpDir, "go", "bin", "kind"), "delete", "cluster")
	os.RemoveAll(c.tmpDir)
}
func (c *testContext) getOS() string {
	osbytes, _, err := c.runCollectingOutput("uname", "-s")
	require.NoError(c.t, err)
	os := string(osbytes)
	switch {
	case strings.HasPrefix(os, "Linux"):
		return "linux"
	case strings.HasPrefix(os, "Darwin"):
		return "darwin"
	default:
		require.FailNow(c.t, fmt.Sprintf("Unknown operating system: '%s'", os))
	}
	return ""
}
var archmap = map[string]string{
	"armv5":   "armv5",
	"armv6":   "armv6",
	"armv7":   "armv7",
	"aarch64": "arm64",
	"x86_64":  "amd64",
	"x86":     "386",
	"i386":    "386",
	"i686":    "386",
}
func (c *testContext) getArch() string {
	archbytes, _, err := c.runCollectingOutput("uname", "-m")
	arch := string(archbytes)
	require.NoError(c.t, err)
	for key, val := range archmap {
		if strings.HasPrefix(arch, key) {
			return val
		}
	}
	require.FailNow(c.t, fmt.Sprintf("Unknown architecture: '%s'", arch))
	return ""
}
func getTestDir(t *testing.T) string {
	testDir, err := os.Getwd()
	require.NoError(t, err)
	return testDir
}
func env(items ...string) []string {
	return items
}
func (c *testContext) applyManagementManifest(manifest string) {
	f, err := ioutil.TempFile(c.tmpDir, "---manifest--*---")
	require.NoError(c.t, err)
	defer os.Remove(f.Name())
	err = ioutil.WriteFile(f.Name(), []byte(manifest), 0600)
	require.NoError(c.t, err)
	c.runAndCheckError("kubectl", "apply", "-f", f.Name())
}
func (c *testContext) applyWorkloadManifest(manifest, kubeconfig string) {
	f, err := ioutil.TempFile(c.tmpDir, "---manifest--*---")
	require.NoError(c.t, err)
	defer os.Remove(f.Name())
	err = ioutil.WriteFile(f.Name(), []byte(manifest), 0600)
	require.NoError(c.t, err)
	_, eout, err := c.runCollectingOutputWithConfig(commandConfig{Env: env("KUBECONFIG=" + kubeconfig)},
		"kubectl", "apply", "-f", f.Name())
	if err != nil {
		require.FailNow(c.t, fmt.Sprintf("Failed to apply manifest: %v -- %s", err, eout))
	}
}
func (c *testContext) run(cmdItems ...string) {
	c.runWithConfig(commandConfig{Stdout: os.Stdout, Stderr: os.Stderr, Env: os.Environ()}, cmdItems...)
}
func (c *testContext) runAndCheckError(cmdItems ...string) {
	c.runWithConfig(commandConfig{Stdout: os.Stdout, Stderr: os.Stderr, Env: os.Environ(), CheckError: true}, cmdItems...)
}
func (c *testContext) runWithConfig(config commandConfig, cmdItems ...string) {
	c.t.Helper()
	cmd := exec.Command(cmdItems[0], cmdItems[1:]...)
	cmd.Dir = config.Dir
	cmd.Stdout = config.Stdout
	cmd.Stderr = config.Stderr
	cmd.Env = config.Env
	err := cmd.Run()
	if config.CheckError {
		if err != nil {
			log.Errorf("Run error: %v", err)
		}
		require.NoError(c.t, err)
	}
}
func (c *testContext) runCollectingOutput(cmdItems ...string) ([]byte, []byte, error) {
	return c.runCollectingOutputWithConfig(commandConfig{Env: os.Environ()}, cmdItems...)
}
func (c *testContext) runCollectingOutputWithConfig(config commandConfig, cmdItems ...string) ([]byte, []byte, error) {
	cmd := exec.Command(cmdItems[0], cmdItems[1:]...)
	var stdout, stderr bytes.Buffer
	cmd.Dir = config.Dir
	cmd.Env = config.Env
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	return stdout.Bytes(), stderr.Bytes(), err
}
func (c *testContext) sshCall(ip, port, cmd string) ([]byte, []byte, error) {
	return c.runCollectingOutput("ssh", "-i", filepath.Join(c.tmpDir, "cluster-key"), "-l", "root",
		"-o", "UserKnownHostsFile /dev/null", "-o", "StrictHostKeyChecking=no", "-p", port, ip, cmd)
}
func (c *testContext) sshAction(ip, port, cmd string) {
	c.runWithConfig(commandConfig{Stdout: os.Stdout, Stderr: os.Stderr, Env: os.Environ(), CheckError: false},
		"ssh", "-i", filepath.Join(c.tmpDir, "cluster-key"), "-l", "root",
		"-o", "UserKnownHostsFile /dev/null", "-o", "StrictHostKeyChecking=no", "-p", port, ip, cmd)
}
func (c *testContext) makeSSHCallAndCheckError(ip, port, cmd string) {
	c.runAndCheckError("ssh", "-i", filepath.Join(c.tmpDir, "cluster-key"), "-l", "root",
		"-o", "UserKnownHostsFile /dev/null", "-o", "StrictHostKeyChecking=no", "-p", port, ip, cmd)
}