func (s *Service) Delete(ctx context.Context) (_ *ctrl.Result, err error) {
	serverList, err := s.listMatchingMachines(ctx)
	if err != nil {
		if checkRateLimitExceeded(err) {
			s.scope.Recorder.Eventf(s.scope.BareMetalMachine, corev1.EventTypeWarning, "HrobotRateLimitExceeded", "Hrobot rate limit exceeded. Wait for %v sec before trying again.", rateLimitTimeOutDeletion)
			return &reconcile.Result{RequeueAfter: rateLimitTimeOutDeletion * time.Second}, nil
		}
		return nil, errors.Wrap(err, "failed to refresh server status")
	}
	server, err := s.findAttachedMachine(serverList)
	if err != nil {
		if checkRateLimitExceeded(err) {
			s.scope.Recorder.Eventf(s.scope.BareMetalMachine, corev1.EventTypeWarning, "HrobotRateLimitExceeded", "Hrobot rate limit exceeded. Wait for %v sec before trying again.", rateLimitTimeOutDeletion)
			return &reconcile.Result{RequeueAfter: rateLimitTimeOutDeletion * time.Second}, nil
		}
		return nil, errors.Wrap(err, "failed to find attached machine")
	}
	if server == nil {
		s.scope.Recorder.Eventf(
			s.scope.BareMetalMachine,
			"UnknownBareMetalMachine",
			"No machine with name %s found to delete",
			s.scope.BareMetalMachine.Name)
		return nil, nil
	}
	_, err = s.scope.HrobotClient().SetBMServerName(server.ServerIP,
		*s.scope.BareMetalMachine.Spec.ServerType+delimiter+"unused-"+s.scope.BareMetalMachine.Name)
	if err != nil {
		return nil, errors.Errorf("Unable to change bare metal server name: ", err)
	}
	return nil, nil
}
In order to avoid the loss of data, all we do is renaming the machine in Hrobot.
After re-attaching the machine we know on which device we installed the image
package baremetal
import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"
	"github.com/cluster-api-provider-hcloud/cluster-api-provider-hcloud/pkg/scope"
	"github.com/cluster-api-provider-hcloud/cluster-api-provider-hcloud/pkg/userdata"
	"github.com/nl2go/hrobot-go/models"
	"github.com/pkg/errors"
	"golang.org/x/crypto/ssh"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/utils/pointer"
	capierrors "sigs.k8s.io/cluster-api/errors"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
const (
	delimiter                      string        = "--"
	hoursBeforeDeletion            time.Duration = 36
	maxWaitingTimeForSSHConnection int           = 200
	defaultPort                    int           = 22
	rateLimitTimeOut               time.Duration = 660
	rateLimitTimeOutDeletion       time.Duration = 120
)
type Service struct {
	scope *scope.BareMetalMachineScope
}
type blockDeviceData struct {
	Name     string `json:"name"`
	Size     string `json:"size"`
	Rotation bool   `json:"rota"`
	Type     string `json:"fstype"`
	Label    string `json:"label"`
}
type blockDevice struct {
	Name     string            `json:"name"`
	Size     string            `json:"size"`
	Rotation bool              `json:"rota"`
	Type     string            `json:"fstype"`
	Label    string            `json:"label"`
	Children []blockDeviceData `json:"children"`
}
type blockDevices struct {
	Devices []blockDevice `json:"blockdevices"`
}
func NewService(scope *scope.BareMetalMachineScope) *Service {
	return &Service{
		scope: scope,
	}
}
func stringSliceContains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}
func (s *Service) Reconcile(ctx context.Context) (_ *ctrl.Result, err error) {
	if s.scope.HcloudCluster.Spec.HrobotTokenRef == nil {
		s.scope.Recorder.Eventf(s.scope.BareMetalMachine, corev1.EventTypeWarning, "NoTokenFound", "No Hrobot token found")
		return nil, errors.Errorf("ERROR: No token for Hetzner Robot provided: Cannot reconcile server %s", s.scope.BareMetalMachine.Name)
	}
	serverList, err := s.listMatchingMachines(ctx)
	if err != nil {
		if checkRateLimitExceeded(err) {
			s.scope.Recorder.Eventf(s.scope.BareMetalMachine, corev1.EventTypeWarning, "HrobotRateLimitExceeded", "Hrobot rate limit exceeded. Wait for %v sec before trying again.", rateLimitTimeOut)
			return &reconcile.Result{RequeueAfter: rateLimitTimeOut * time.Second}, nil
		}
		return nil, errors.Wrap(err, "failed to list machines")
	}
	actualServer, err := s.findAttachedMachine(serverList)
	if err != nil {
		return nil, errors.Wrap(err, "failed to find attached machine")
	}
	if actualServer != nil {
		if actualServer.Cancelled {
			s.scope.V(1).Info("server has been cancelled", "server", actualServer.ServerName, "cancelled",
				actualServer.Cancelled, "paid until", actualServer.PaidUntil)
			paidUntil, err := time.Parse("2006-01-02", actualServer.PaidUntil)
			if err != nil {
				return nil, errors.Errorf("ERROR: Failed to parse paidUntil date. Error: %s", err)
			}
			if paidUntil.Before(time.Now().Add(hoursBeforeDeletion * time.Hour)) {
				s.scope.Recorder.Eventf(
					s.scope.BareMetalMachine,
					corev1.EventTypeWarning,
					"CancelledBareMetalMachine",
					"Bare metal machine is cancelled and paid until %s",
					actualServer.PaidUntil)
				er := capierrors.UpdateMachineError
				s.scope.BareMetalMachine.Status.FailureReason = &er
				s.scope.BareMetalMachine.Status.FailureMessage = pointer.StringPtr("Machine has been cancelled and is paid until less than" + hoursBeforeDeletion.String() + "hours")
				s.scope.BareMetalMachine.Status.Ready = false
			}
		}
	} else {
		newServer, err := s.findNewMachine(serverList)
		if err != nil {
			return nil, errors.Errorf("Failed to find new machine: %s", err)
		}
		if newServer.Status != "ready" {
			s.scope.V(2).Info("server not in running state", "server", newServer.ServerName, "status", newServer.Status)
			return &reconcile.Result{RequeueAfter: 300 * time.Second}, nil
		}
		if !s.scope.IsBootstrapDataReady(ctx) {
			s.scope.V(2).Info("Bootstrap data is not ready yet")
			return &reconcile.Result{RequeueAfter: 15 * time.Second}, nil
		}
		s.scope.BareMetalMachine.Status.ServerState = "initializing"
		if err := s.provisionMachine(ctx, *newServer); err != nil {
			if checkRateLimitExceeded(err) {
				s.scope.Recorder.Eventf(s.scope.BareMetalMachine, corev1.EventTypeWarning, "HrobotRateLimitExceeded", "Hrobot rate limit exceeded. Wait for %v sec before trying again.", rateLimitTimeOut)
				return &reconcile.Result{RequeueAfter: rateLimitTimeOut * time.Second}, nil
			}
			return nil, errors.Errorf("Failed to provision new machine: %s", err)
		}
		actualServer = newServer
	}
	providerID := fmt.Sprintf("hcloud://%d", actualServer.ServerNumber)
	s.scope.BareMetalMachine.Status.ServerState = "running"
	s.scope.BareMetalMachine.Spec.ProviderID = &providerID
	s.scope.BareMetalMachine.Status.Ready = true
	return nil, nil
}
func (s *Service) findAttachedMachine(servers []models.Server) (*models.Server, error) {
	var actualServer models.Server
	if len(servers) == 0 {
		return nil, nil
	}
	var check int
	for _, server := range servers {
		splitName := strings.Split(server.ServerName, delimiter)
		if len(splitName) == 3 && splitName[2] == s.scope.BareMetalMachine.Name {
			actualServer = server
			check++
		}
	}
	if check > 1 {
		s.scope.Recorder.Eventf(
			s.scope.BareMetalMachine,
			corev1.EventTypeWarning,
			"MultipleBareMetalMachines",
			"Found %v bare metal machines of the name %s attached to the cluster",
			check, actualServer.ServerName)
		return nil, errors.Errorf("There are %s servers which are attached to the cluster with name %s", check, actualServer.ServerName)
	} else if check == 0 {
		return nil, nil
	} else {
		return &actualServer, nil
	}
}
func (s *Service) findNewMachine(servers []models.Server) (*models.Server, error) {
	if len(servers) == 0 {
		s.scope.Recorder.Eventf(
			s.scope.BareMetalMachine,
			corev1.EventTypeWarning,
			"EmptyListBareMetalMachines",
			"No machines of type %s found",
			*s.scope.BareMetalMachine.Spec.ServerType)
		return nil, errors.Errorf("No bare metal server found with type %s", *s.scope.BareMetalMachine.Spec.ServerType)
	}
	for _, server := range servers {
		splitName := strings.Split(server.ServerName, delimiter)
		if len(splitName) == 2 {
			return &server, nil
		}
	}
	s.scope.Recorder.Eventf(
		s.scope.BareMetalMachine,
		corev1.EventTypeWarning,
		"NoAvailableBareMetalMachines",
		"No machines of type %s are available",
		*s.scope.BareMetalMachine.Spec.ServerType)
	return nil, errors.Errorf("There are no available servers of type %s left", *s.scope.BareMetalMachine.Spec.ServerType)
}
func (s *Service) provisionMachine(ctx context.Context, server models.Server) error {
	s.scope.V(4).Info("Started provisioning bare metal machine")
	sshKeyName, _, privateSSHKey, err := s.retrieveSSHSecret(ctx)
	if err != nil {
		return errors.Errorf("Unable to retrieve SSH secret: ", err)
	}
	sshFingerprint, err := s.getSSHFingerprintFromName(sshKeyName)
	if err != nil {
		return errors.Errorf("Unable to get SSH fingerprint for the SSH key %s: %s ", sshKeyName, err)
	}
	userDataInitial, err := s.scope.GetRawBootstrapData(ctx)
	if err != nil {
		return err
	}
	userData, err := userdata.NewFromReader(bytes.NewReader(userDataInitial))
	if err != nil {
		return err
	}
	kubeadmConfig, err := userData.GetKubeadmConfig()
	if err != nil {
		return err
	}
	cloudProviderKey := "cloud-provider"
	cloudProviderValue := "external"
	if j := kubeadmConfig.JoinConfiguration; j != nil {
		if j.NodeRegistration.KubeletExtraArgs == nil {
			j.NodeRegistration.KubeletExtraArgs = make(map[string]string)
		}
		if _, ok := j.NodeRegistration.KubeletExtraArgs[cloudProviderKey]; !ok {
			j.NodeRegistration.KubeletExtraArgs[cloudProviderKey] = cloudProviderValue
		}
	}
	if err := userData.SetKubeadmConfig(kubeadmConfig); err != nil {
		return err
	}
	userDataBytes := bytes.NewBuffer(nil)
	if err := userData.WriteYAML(userDataBytes); err != nil {
		return errors.Errorf("Error while writing yaml file", err)
	}
	cloudInitConfigString := userDataBytes.String()
	s.scope.V(4).Info("Activate rescue")
	_, err = s.scope.HrobotClient().ActivateRescue(server.ServerIP, sshFingerprint)
	if err != nil {
		return errors.Errorf("Unable to activate rescue system: ", err)
	}
	s.scope.V(4).Info("Reset machine")
	_, err = s.scope.HrobotClient().ResetBMServer(server.ServerIP, "hw")
	s.scope.V(4).Info("Run SSH command hostname")
	stdout, stderr, err := runSSH("hostname", server.ServerIP, 22, privateSSHKey)
	if err != nil {
		return errors.Errorf("SSH command hostname returned the error %s. The output of stderr is %s", err, stderr)
	}
	if !strings.Contains(stdout, "rescue") {
		return errors.Errorf("Rescue system not successfully started. Output of command hostname is %s", stdout)
	}
	s.scope.V(4).Info("Get block devices")
	var blockDevices blockDevices
	blockDeviceCommand := "lsblk -o name,size,rota,fstype,label -e1 -e7 --json"
	stdout, stderr, err = runSSH(blockDeviceCommand, server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		return errors.Errorf("Error running the ssh command %s: Error: %s, stderr: %s", blockDeviceCommand, err, stderr)
	}
	err = json.Unmarshal([]byte(stdout), &blockDevices)
	if err != nil {
		return errors.Errorf("Error while unmarshaling: %s", err)
	}
	drive, err := findCorrectDevice(blockDevices)
	if err != nil {
		return errors.Errorf("Error while finding correct device: %s", err)
	}
	partitionString := `PART /boot ext3 512M
PART / ext4 all`
	if s.scope.BareMetalMachine.Spec.Partition != nil {
		partitionString = *s.scope.BareMetalMachine.Spec.Partition
	}
	autoSetup := fmt.Sprintf(
		`cat > /autosetup << EOF
DRIVE1 /dev/%s
BOOTLOADER grub
HOSTNAME %s
%s
IMAGE %s
EOF`,
		drive, s.scope.Cluster.Name+delimiter+*s.scope.BareMetalMachine.Spec.ServerType+delimiter+s.scope.BareMetalMachine.Name, partitionString, *s.scope.BareMetalMachine.Spec.ImagePath)
	s.scope.V(4).Info("Send auto setup file to server")
	_, stderr, err = runSSH(autoSetup, server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		return errors.Errorf("SSH command autosetup returned the error %s. The output of stderr is %s", err, stderr)
	}
	s.scope.V(4).Info("Install image")
	_, stderr, err = runSSH("bash /root/.oldroot/nfs/install/installimage", server.ServerIP, 22, privateSSHKey)
	if err != nil {
		wipeCommand := fmt.Sprintf("wipefs -a /dev/%s", drive)
		_, _, _ = runSSH(wipeCommand, server.ServerIP, defaultPort, privateSSHKey)
		return errors.Errorf("SSH command installimage returned the error %s. The output of stderr is %s", err, stderr)
	}
	s.scope.V(4).Info("Get block devices")
	stdout, stderr, err = runSSH(blockDeviceCommand, server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		wipeCommand := fmt.Sprintf("wipefs -a /dev/%s", drive)
		_, _, _ = runSSH(wipeCommand, server.ServerIP, defaultPort, privateSSHKey)
		return errors.Errorf("Error running the ssh command %s: Error: %s, stderr: %s", blockDeviceCommand, err, stderr)
	}
	err = json.Unmarshal([]byte(stdout), &blockDevices)
	if err != nil {
		return errors.Errorf("Error while unmarshaling: %s", err)
	}
	command, err := labelChildrenCommand(blockDevices, drive)
	if err != nil {
		return errors.Errorf("Error while constructing labeling children command of device %s: %s", drive, err)
	}
	s.scope.V(4).Info("Label children")
	_, stderr, err = runSSH(command, server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		wipeCommand := fmt.Sprintf("wipefs -a /dev/%s", drive)
		_, _, _ = runSSH(wipeCommand, server.ServerIP, defaultPort, privateSSHKey)
		return errors.Errorf("Error running the ssh command %s: Error: %s, stderr: %s", command, err, stderr)
	}
	_, stderr, err = runSSH("sleep 30", server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		return errors.Errorf("Error running the ssh command sleep 30: Error: %s, stderr: %s", err, stderr)
	}
	s.scope.V(4).Info("Reboot")
	_, stderr, err = runSSH("reboot", server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		if !strings.Contains(err.Error(), "exited without exit status or exit signal") {
			return errors.Errorf("Error running the ssh command reboot: Error: %s, stderr: %s", err, stderr)
		}
	}
	_, stderr, err = runSSH("sleep 60", server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		return errors.Errorf("Connection to server after reboot could not be established. Error: %s, stderr: %s", err, stderr)
	}
	s.scope.V(4).Info("Create nocloud directory")
	command = "mkdir -p /var/lib/cloud/seed/nocloud"
	_, stderr, err = runSSH(command, server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		return errors.Errorf("Error running the ssh command %s: Error: %s, stderr: %s", command, err, stderr)
	}
	s.scope.V(4).Info("Create meta data for cloud init")
	command = "echo 'instance-id: iid-system-uuid' >> /var/lib/cloud/seed/nocloud/meta-data"
	_, stderr, err = runSSH(command, server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		return errors.Errorf("Error running the ssh command %s: Error: %s, stderr: %s", command, err, stderr)
	}
	cloudInitCommand := fmt.Sprintf(
		`cat > /var/lib/cloud/seed/nocloud/user-data << EOF
%s
EOF`, cloudInitConfigString)
	s.scope.V(4).Info("Send userdata")
	_, stderr, err = runSSH(cloudInitCommand, server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		return errors.Errorf("Error running the ssh command %s: Error: %s, stderr: %s", cloudInitCommand, err, stderr)
	}
	_, stderr, err = runSSH("sleep 30", server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		return errors.Errorf("Connection to server after reboot could not be established. Error: %s, stderr: %s", err, stderr)
	}
	s.scope.V(4).Info("Reboot")
	_, stderr, err = runSSH("reboot", server.ServerIP, defaultPort, privateSSHKey)
	if err != nil {
		if !strings.Contains(err.Error(), "exited without exit status or exit signal") {
			return errors.Errorf("Error running the ssh command reboot: Error: %s, stderr: %s", err, stderr)
		}
	}
	s.scope.V(4).Info("Set server name and finish")
	_, err = s.scope.HrobotClient().SetBMServerName(server.ServerIP,
		s.scope.Cluster.Name+delimiter+*s.scope.BareMetalMachine.Spec.ServerType+delimiter+s.scope.BareMetalMachine.Name)
	if err != nil {
		return errors.Errorf("Unable to change bare metal server name: ", err)
	}
	s.scope.Recorder.Eventf(
		s.scope.BareMetalMachine,
		"CreateBareMetalMachine",
		"Created bare metal machine %s",
		s.scope.BareMetalMachine.Name)
	return nil
}
func checkRateLimitExceeded(err error) bool {
	return strings.Contains(err.Error(), "rate limit exceeded")
}