func (g *gravity) ExecScript(ctx context.Context, scriptUrl string, args []string) error {
	log := g.Logger().WithFields(logrus.Fields{
		"script": scriptUrl, "args": args})
	log.Debug("Execute.")
	spath, err := sshutils.TransferFile(ctx, g.Client(), log,
		scriptUrl, defaults.TmpDir, g.param.env)
	if err != nil {
		log.WithError(err).Error("failed to transfer script")
		return trace.Wrap(err)
	}
	err = sshutils.Run(ctx, g.Client(), log,
		fmt.Sprintf("sudo /bin/bash -x %s %s", spath, strings.Join(args, " ")), nil)
	return trace.Wrap(err)
}
ExecScript will transfer and execute script provided with given args
package gravity
import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"
	"time"
	"github.com/gravitational/robotest/infra"
	"github.com/gravitational/robotest/lib/constants"
	"github.com/gravitational/robotest/lib/defaults"
	sshutils "github.com/gravitational/robotest/lib/ssh"
	"github.com/gravitational/robotest/lib/wait"
	"github.com/cenkalti/backoff"
	"github.com/gravitational/trace"
	"github.com/sirupsen/logrus"
	"golang.org/x/crypto/ssh"
)
type Gravity interface {
	json.Marshaler
	fmt.Stringer
	SetInstaller(ctx context.Context, installerUrl, subdir string) error
	TransferFile(ctx context.Context, url, subdir string) error
	ExecScript(ctx context.Context, scriptUrl string, args []string) error
	Install(ctx context.Context, param InstallParam) error
	Status(ctx context.Context) (*GravityStatus, error)
	OfflineUpdate(ctx context.Context, installerUrl string) error
	Join(ctx context.Context, param JoinCmd) error
	Leave(ctx context.Context, graceful Graceful) error
	Remove(ctx context.Context, node string, graceful Graceful) error
	Uninstall(ctx context.Context) error
	UninstallApp(ctx context.Context) error
	PowerOff(ctx context.Context, graceful Graceful) error
	Reboot(ctx context.Context, graceful Graceful) error
	CollectLogs(ctx context.Context, prefix string, args ...string) (localPath string, err error)
	Upload(ctx context.Context) error
	Upgrade(ctx context.Context) error
	RunInPlanet(ctx context.Context, cmd string, args ...string) (string, error)
	Node() infra.Node
	Offline() bool
	Client() *ssh.Client
	Logger() logrus.FieldLogger
}
type Graceful bool
type InstallParam struct {
	Token string `json:"-"`
	Role string `json:"role" validate:"required"`
	Cluster string `json:"cluster"`
	Flavor string `json:"flavor" validate:"required"`
	EnableRemoteSupport bool `json:"remote_support"`
	CloudProvider string `json:"cloud_provider,omitempty"`
	GCENodeTag string `json:"gce_node_tag,omitempty"`
	StateDir string `json:"state_dir" validate:"required"`
	OSFlavor OS `json:"os" validate:"required"`
	DockerStorageDriver StorageDriver `json:"storage_driver"`
	InstallerURL string `json:"installer_url,omitempty"`
	OpsAdvertiseAddr string `json:"ops_advertise_addr,omitempty"`
	ServiceUID *uint `json:"service_uid,omitempty"`
	ServiceGID *uint `json:"service_gid,omitempty"`
}
type JoinCmd struct {
	InstallDir string
	PeerAddr string
	Token string
	Role string
	StateDir string
}
type gravity struct {
	node       infra.Node
	ssh        *ssh.Client
	installDir string
	param      cloudDynamicParams
	ts         time.Time
	log        logrus.FieldLogger
}
func (g *gravity) MarshalJSON() ([]byte, error) {
	return json.Marshal(map[string]string{
		"public_ip": g.node.Addr(),
		"ip":        g.node.PrivateAddr(),
	})
}
func sshClient(ctx context.Context, node infra.Node, log logrus.FieldLogger) (*ssh.Client, error) {
	ctx, cancel := context.WithTimeout(ctx, deadlineSSH)
	defer cancel()
	var client *ssh.Client
	b := backoff.NewConstantBackOff(retrySSH)
	err := wait.RetryWithInterval(ctx, b, func() (err error) {
		client, err = node.Client()
		if err == nil {
			log.Debug("Connected via SSH.")
			return nil
		}
		log.WithError(err).Debug("Waiting for SSH.")
		return trace.Wrap(err)
	}, log)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	return client, nil
}
func (g *gravity) Logger() logrus.FieldLogger {
	return g.log
}
func (g *gravity) String() string {
	return fmt.Sprintf("node(private_addr=%s, public_addr=%s)",
		g.node.PrivateAddr(), g.node.Addr())
}
func (g *gravity) Node() infra.Node {
	return g.node
}
func (g *gravity) Client() *ssh.Client {
	return g.ssh
}
func (g *gravity) Install(ctx context.Context, param InstallParam) error {
	type cmd struct {
		InstallDir    string
		PrivateAddr   string
		DockerDevice  string
		StorageDriver string
		AgentLogPath  string
		InstallParam
	}
	dockerDevice := g.param.dockerDevice
	if g.param.storageDriver != constants.DeviceMapper {
		dockerDevice = ""
	}
	config := cmd{
		InstallDir:    g.installDir,
		PrivateAddr:   g.Node().PrivateAddr(),
		DockerDevice:  dockerDevice,
		StorageDriver: g.param.storageDriver.Driver(),
		AgentLogPath:  defaults.AgentLogPath,
		InstallParam:  param,
	}
	var buf bytes.Buffer
	err := installCmdTemplate.Execute(&buf, config)
	if err != nil {
		return trace.Wrap(err, buf.String())
	}
	err = sshutils.Run(ctx, g.Client(), g.Logger(), buf.String(),
		map[string]string{
			constants.GravitySELinuxEnv: "true",
		})
	return trace.Wrap(err, param)
}
var installCmdTemplate = template.Must(
	template.New("gravity_install").Parse(`
		cd {{.InstallDir}} && ./gravity version && sudo ./gravity install --debug \
		--advertise-addr={{.PrivateAddr}} --token={{.Token}} --flavor={{.Flavor}} \
		{{if .DockerDevice}}--docker-device={{.DockerDevice}}{{end}} \
		{{if .StorageDriver}}--storage-driver={{.StorageDriver}}{{end}} \
		--system-log-file={{ .AgentLogPath }} \
		--cloud-provider=generic --state-dir={{.StateDir}} \
		--httpprofile=localhost:6061 \
		{{if .Cluster}}--cluster={{.Cluster}}{{end}} \
		{{if .OpsAdvertiseAddr}}--ops-advertise-addr={{.OpsAdvertiseAddr}}{{end}}\
		{{if .ServiceUID}}--service-uid={{.ServiceUID}}{{end}}\
		{{if .ServiceGID}}--service-gid={{.ServiceGID}}{{end}}\
`))
func (g *gravity) Status(ctx context.Context) (*GravityStatus, error) {
	cmd := fmt.Sprintf("sudo gravity status --output=json --system-log-file=%v",
		defaults.AgentLogPath)
	status := GravityStatus{}
	err := sshutils.RunAndParse(ctx, g.Client(), g.Logger(), cmd, nil, parseStatus(&status))
	if err != nil {
		if exitErr, ok := trace.Unwrap(err).(sshutils.ExitStatusError); ok {
			g.Logger().WithFields(logrus.Fields{
				"private_addr": g.Node().PrivateAddr(),
				"addr":         g.Node().Addr(),
				"command":      cmd,
				"exit code":    exitErr.ExitStatus(),
			}).Warn("Failed.")
		}
		return nil, trace.Wrap(err, cmd)
	}
	return &status, nil
}
func (g *gravity) OfflineUpdate(ctx context.Context, installerUrl string) error {
	return nil
}
func (g *gravity) Join(ctx context.Context, param JoinCmd) error {
	type cmd struct {
		InstallDir   string
		PrivateAddr  string
		DockerDevice string
		AgentLogPath string
		JoinCmd
	}
	dockerDevice := g.param.dockerDevice
	if g.param.storageDriver != constants.DeviceMapper {
		dockerDevice = ""
	}
	var buf bytes.Buffer
	err := joinCmdTemplate.Execute(&buf, cmd{
		InstallDir:   g.installDir,
		PrivateAddr:  g.Node().PrivateAddr(),
		DockerDevice: dockerDevice,
		AgentLogPath: defaults.AgentLogPath,
		JoinCmd:      param,
	})
	if err != nil {
		return trace.Wrap(err, buf.String())
	}
	err = sshutils.Run(ctx, g.Client(), g.Logger(), buf.String(),
		map[string]string{
			constants.GravitySELinuxEnv: "true",
		})
	return trace.Wrap(err, param)
}
var joinCmdTemplate = template.Must(
	template.New("gravity_join").Parse(`
		cd {{.InstallDir}} && sudo ./gravity join {{.PeerAddr}} \
		--advertise-addr={{.PrivateAddr}} --token={{.Token}} --debug \
		--role={{.Role}} --docker-device={{.DockerDevice}} \
		--system-log-file={{.AgentLogPath}} --state-dir={{.StateDir}} \
		--httpprofile=localhost:6061`))
func (g *gravity) Leave(ctx context.Context, graceful Graceful) error {
	var cmd string
	if graceful {
		cmd = `leave --confirm`
	} else {
		cmd = `leave --confirm --force`
	}
	return trace.Wrap(g.runOp(ctx, cmd, nil))
}
func (g *gravity) Remove(ctx context.Context, node string, graceful Graceful) error {
	var cmd string
	if graceful {
		cmd = fmt.Sprintf(`remove --confirm %s`, node)
	} else {
		cmd = fmt.Sprintf(`remove --confirm --force %s`, node)
	}
	return trace.Wrap(g.runOp(ctx, cmd, nil))
}
func (g *gravity) Uninstall(ctx context.Context) error {
	cmd := fmt.Sprintf(`cd %s && sudo ./gravity system uninstall --confirm --system-log-file=%v`,
		g.installDir, defaults.AgentLogPath)
	err := sshutils.Run(ctx, g.Client(), g.Logger(), cmd, nil)
	return trace.Wrap(err, cmd)
}
func (g *gravity) UninstallApp(ctx context.Context) error {
	cmd := fmt.Sprintf("cd %s && sudo ./gravity app uninstall $(./gravity app-package) --system-log-file=%v",
		g.installDir, defaults.AgentLogPath)
	err := sshutils.Run(ctx, g.Client(), g.Logger(), cmd, nil)
	return trace.Wrap(err, cmd)
}
func (g *gravity) PowerOff(ctx context.Context, graceful Graceful) error {
	var cmd string
	if graceful {
		cmd = "sudo shutdown -h now"
	} else {
		cmd = "sudo poweroff -f"
	}
	err := sshutils.RunAndParse(ctx, g.Client(), g.Logger(), cmd, nil, nil)
	if err != nil {
		return trace.Wrap(err)
	}
	g.ssh = nil
	return nil
}
func (g *gravity) Offline() bool {
	return g.ssh == nil
}
func (g *gravity) Reboot(ctx context.Context, graceful Graceful) error {
	var cmd string
	if graceful {
		cmd = "sudo shutdown -r now"
	} else {
		cmd = "sudo reboot -f"
	}
	err := sshutils.RunAndParse(ctx, g.Client(), g.Logger(), cmd, nil, nil)
	if err != nil {
		return trace.Wrap(err)
	}
	client, err := sshClient(ctx, g.Node(), g.Logger())
	if err != nil {
		return trace.Wrap(err, "SSH reconnect")
	}
	g.ssh = client
	return nil
}
func (g *gravity) CollectLogs(ctx context.Context, prefix string, args ...string) (localPath string, err error) {
	if g.ssh == nil {
		return "", trace.AccessDenied("cannot collect logs from an offline node %v", g)
	}
	localPath = filepath.Join(g.param.StateDir, "node-logs", prefix,
		fmt.Sprintf("%v-logs.tgz", g.Node().PrivateAddr()))
	return localPath, trace.Wrap(sshutils.PipeCommand(ctx, g.Client(), g.Logger(),
		fmt.Sprintf("cd %v && sudo ./gravity system report %v", g.installDir,
			strings.Join(args, " ")), localPath))
}
func (g *gravity) SetInstaller(ctx context.Context, installerURL string, subdir string) error {
	installDir := filepath.Join(g.param.homeDir, subdir)
	log := g.Logger().WithFields(logrus.Fields{"installer_url": installerURL, "installer_dir": installDir})
	log.Infof("Transfer installer %v -> %v.", installerURL, installDir)
	tgz, err := sshutils.TransferFile(ctx, g.Client(), log, installerURL, installDir, g.param.env)
	if err != nil {
		log.WithError(err).Warnf("Failed to transfer installer %v -> %v.", installerURL, installDir)
		return trace.Wrap(err)
	}
	err = sshutils.Run(ctx, g.Client(), log, fmt.Sprintf("tar -xvf %s -C %s", tgz, installDir), nil)
	if err != nil {
		return trace.Wrap(err)
	}
	g.installDir = installDir
	return nil
}
func (g *gravity) TransferFile(ctx context.Context, url, subdir string) error {
	dir := filepath.Join(g.param.homeDir, subdir)
	log := g.Logger().WithFields(logrus.Fields{"url": url, "dir": dir})
	log.Infof("Transfer %v -> %v.", url, dir)
	_, err := sshutils.TransferFile(ctx, g.Client(), log, url, dir, g.param.env)
	if err != nil {
		log.WithError(err).Warnf("Failed to transfer file %v -> %v.", url, dir)
		return trace.Wrap(err)
	}
	g.installDir = dir
	return nil
}