<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package virthandler

import (
	"context"
	goerror "errors"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/opencontainers/runc/libcontainer/cgroups"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	netcache "kubevirt.io/kubevirt/pkg/network/cache"
	"kubevirt.io/kubevirt/pkg/util"

	"kubevirt.io/kubevirt/pkg/virt-handler/heartbeat"

	"kubevirt.io/kubevirt/pkg/util/migrations"

	container_disk "kubevirt.io/kubevirt/pkg/virt-handler/container-disk"
	device_manager "kubevirt.io/kubevirt/pkg/virt-handler/device-manager"
	hotplug_volume "kubevirt.io/kubevirt/pkg/virt-handler/hotplug-disk"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/client-go/log"
	"kubevirt.io/kubevirt/pkg/controller"
	diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils"
	cmdv1 "kubevirt.io/kubevirt/pkg/handler-launcher-com/cmd/v1"
	hostdisk "kubevirt.io/kubevirt/pkg/host-disk"
	neterrors "kubevirt.io/kubevirt/pkg/network/errors"
	pvcutils "kubevirt.io/kubevirt/pkg/util/types"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
	virtcache "kubevirt.io/kubevirt/pkg/virt-handler/cache"
	cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
	"kubevirt.io/kubevirt/pkg/virt-handler/isolation"
	migrationproxy "kubevirt.io/kubevirt/pkg/virt-handler/migration-proxy"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/network"
	"kubevirt.io/kubevirt/pkg/watchdog"
)

const (
	//VolumeReadyReason is the reason set when the volume is ready.
	VolumeReadyReason = "VolumeReady"
	//VolumeUnMountedFromPodReason is the reason set when the volume is unmounted from the virtlauncher pod
	VolumeUnMountedFromPodReason = "VolumeUnMountedFromPod"
	//VolumeMountedToPodReason is the reason set when the volume is mounted to the virtlauncher pod
	VolumeMountedToPodReason = "VolumeMountedToPod"
	//VolumeUnplugged is the reason set when the volume is completely unplugged from the VMI
	VolumeUnplugged = "VolumeUnplugged"
	//VMIDefined is the reason set when a VMI is defined
	VMIDefined = "VirtualMachineInstance defined."
	//VMIStarted is the reason set when a VMI is started
	VMIStarted = "VirtualMachineInstance started."
	//VMIShutdown is the reason set when a VMI is shutdown
	VMIShutdown = "The VirtualMachineInstance was shut down."
	//VMICrashed is the reason set when a VMI crashed
	VMICrashed = "The VirtualMachineInstance crashed."
	//VMIAbortingMigration is the reason set when migration is being aborted
	VMIAbortingMigration = "VirtualMachineInstance is aborting migration."
	//VMIMigrating in the reason set when the VMI is migrating
	VMIMigrating = "VirtualMachineInstance is migrating."
	//VMIMigrationTargetPrepared is the reason set when the migration target has been prepared
	VMIMigrationTargetPrepared = "VirtualMachineInstance Migration Target Prepared."
	//VMIStopping is the reason set when the VMI is stopping
	VMIStopping = "VirtualMachineInstance stopping"
	//VMIGracefulShutdown is the reason set when the VMI is gracefully shut down
	VMIGracefulShutdown = "Signaled Graceful Shutdown"
	//VMISignalDeletion is the reason set when the VMI has signal deletion
	VMISignalDeletion = "Signaled Deletion"
)

var RequiredGuestAgentCommands = []string{
	"guest-ping",
	"guest-get-time",
	"guest-info",
	"guest-shutdown",
	"guest-network-get-interfaces",
	"guest-get-fsinfo",
	"guest-get-host-name",
	"guest-get-users",
	"guest-get-timezone",
	"guest-get-osinfo",
}

var SSHRelatedGuestAgentCommands = []string{
	"guest-exec-status",
	"guest-exec",
	"guest-file-open",
	"guest-file-close",
	"guest-file-read",
	"guest-file-write",
}

var PasswordRelatedGuestAgentCommands = []string{
	"guest-<PASSWORD>-user-password",
}

type launcherClientInfo struct {
	client              cmdclient.LauncherClient
	socketFile          string
	domainPipeStopChan  chan struct{}
	notInitializedSince time.Time
	ready               bool
}

func NewController(
	recorder record.EventRecorder,
	clientset kubecli.KubevirtClient,
	host string,
	ipAddress string,
	virtShareDir string,
	virtPrivateDir string,
	vmiSourceInformer cache.SharedIndexInformer,
	vmiTargetInformer cache.SharedIndexInformer,
	domainInformer cache.SharedInformer,
	gracefulShutdownInformer cache.SharedIndexInformer,
	watchdogTimeoutSeconds int,
	maxDevices int,
	clusterConfig *virtconfig.ClusterConfig,
	podIsolationDetector isolation.PodIsolationDetector,
	migrationProxy migrationproxy.ProxyManager,
) *VirtualMachineController {

	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

	c := &VirtualMachineController{
		Queue:                       queue,
		recorder:                    recorder,
		clientset:                   clientset,
		host:                        host,
		ipAddress:                   ipAddress,
		virtShareDir:                virtShareDir,
		vmiSourceInformer:           vmiSourceInformer,
		vmiTargetInformer:           vmiTargetInformer,
		domainInformer:              domainInformer,
		gracefulShutdownInformer:    gracefulShutdownInformer,
		heartBeatInterval:           1 * time.Minute,
		watchdogTimeoutSeconds:      watchdogTimeoutSeconds,
		migrationProxy:              migrationProxy,
		podIsolationDetector:        podIsolationDetector,
		containerDiskMounter:        container_disk.NewMounter(podIsolationDetector, virtPrivateDir+"/container-disk-mount-state"),
		hotplugVolumeMounter:        hotplug_volume.NewVolumeMounter(podIsolationDetector, virtPrivateDir+"/hotplug-volume-mount-state"),
		clusterConfig:               clusterConfig,
		networkCacheStoreFactory:    netcache.NewInterfaceCacheFactory(),
		virtLauncherFSRunDirPattern: "/proc/%d/root/var/run",
	}

	vmiSourceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addFunc,
		DeleteFunc: c.deleteFunc,
		UpdateFunc: c.updateFunc,
	})

	vmiTargetInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addFunc,
		DeleteFunc: c.deleteFunc,
		UpdateFunc: c.updateFunc,
	})

	domainInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addDomainFunc,
		DeleteFunc: c.deleteDomainFunc,
		UpdateFunc: c.updateDomainFunc,
	})

	gracefulShutdownInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addFunc,
		DeleteFunc: c.deleteFunc,
		UpdateFunc: c.updateFunc,
	})

	c.launcherClients = make(map[types.UID]*launcherClientInfo)
	c.phase1NetworkSetupCache = make(map[types.UID]int)
	c.podInterfaceCache = make(map[string]*netcache.PodCacheInterface)

	c.domainNotifyPipes = make(map[string]string)

	permissions := "rw"
	if cgroups.IsCgroup2UnifiedMode() {
		// Need 'rwm' permissions otherwise ebpf filtering program attached by runc
		// will deny probing the device file with 'access' syscall. That in turn
		// will lead to libvirtd failure on VM startup.
		// This has been fixed upstream:
		//   https://github.com/opencontainers/runc/pull/2796
		// but the workaround is still needed to support previous versions without
		// the patch.
		permissions = "rwm"
	}

	c.deviceManagerController = device_manager.NewDeviceController(c.host, maxDevices, permissions, clusterConfig)
	c.heartBeat = heartbeat.NewHeartBeat(clientset.CoreV1(), c.deviceManagerController, clusterConfig, host)

	return c
}

type VirtualMachineController struct {
	recorder                 record.EventRecorder
	clientset                kubecli.KubevirtClient
	host                     string
	ipAddress                string
	virtShareDir             string
	virtPrivateDir           string
	Queue                    workqueue.RateLimitingInterface
	vmiSourceInformer        cache.SharedIndexInformer
	vmiTargetInformer        cache.SharedIndexInformer
	domainInformer           cache.SharedInformer
	gracefulShutdownInformer cache.SharedIndexInformer
	launcherClients          map[types.UID]*launcherClientInfo
	launcherClientLock       sync.Mutex
	heartBeatInterval        time.Duration
	watchdogTimeoutSeconds   int
	deviceManagerController  *device_manager.DeviceController
	migrationProxy           migrationproxy.ProxyManager
	podIsolationDetector     isolation.PodIsolationDetector
	containerDiskMounter     container_disk.Mounter
	hotplugVolumeMounter     hotplug_volume.VolumeMounter
	clusterConfig            *virtconfig.ClusterConfig

	// records if pod network phase1 has completed
	// phase1 involves cycling an entire posix thread
	// so for performance, knowing phase1 is complete
	// prevents cycling an unncessary posix thread.
	phase1NetworkSetupCache     map[types.UID]int
	phase1NetworkSetupCacheLock sync.Mutex

	// key is the file path, value is the contents.
	// if key exists, then don't read directly from file.
	podInterfaceCache     map[string]*netcache.PodCacheInterface
	podInterfaceCacheLock sync.Mutex

	domainNotifyPipes           map[string]string
	networkCacheStoreFactory    netcache.InterfaceCacheFactory
	virtLauncherFSRunDirPattern string
	heartBeat                   *heartbeat.HeartBeat
}

type virtLauncherCriticalNetworkError struct {
	msg string
}

func (e *virtLauncherCriticalNetworkError) Error() string { return e.msg }

type virtLauncherCriticalSecurebootError struct {
	msg string
}

func (e *virtLauncherCriticalSecurebootError) Error() string { return e.msg }

func handleDomainNotifyPipe(domainPipeStopChan chan struct{}, ln net.Listener, virtShareDir string, vmi *v1.VirtualMachineInstance) {

	fdChan := make(chan net.Conn, 100)

	// Listen for new connections,
	// Close listener and exit when stop encountered
	go func(vmi *v1.VirtualMachineInstance, ln net.Listener, domainPipeStopChan chan struct{}) {
		for {
			select {
			case <-domainPipeStopChan:
				log.Log.Object(vmi).Infof("closing notify pipe listener for vmi")
				ln.Close()
				return
			default:
				fd, err := ln.Accept()
				if err != nil {
					log.Log.Reason(err).Error("Domain pipe accept error encountered.")
					// keep listening until stop invoked
					time.Sleep(1)
				} else {
					fdChan <- fd
				}
			}
		}
	}(vmi, ln, domainPipeStopChan)

	// Process new connections
	// exit when stop encountered
	go func(vmi *v1.VirtualMachineInstance, fdChan chan net.Conn, domainPipeStopChan chan struct{}) {
		for {
			select {
			case <-domainPipeStopChan:
				return
			case fd := <-fdChan:
				go func(vmi *v1.VirtualMachineInstance) {
					defer fd.Close()

					// pipe the VMI domain-notify.sock to the virt-handler domain-notify.sock
					// so virt-handler receives notifications from the VMI
					conn, err := net.Dial("unix", filepath.Join(virtShareDir, "domain-notify.sock"))
					if err != nil {
						log.Log.Reason(err).Error("error connecting to domain-notify.sock for proxy connection")
						return
					}
					defer conn.Close()

					log.Log.Object(vmi).Infof("Accepted new notify pipe connection for vmi")
					copyErr := make(chan error, 2)
					go func() {
						_, err := io.Copy(fd, conn)
						copyErr <- err
					}()
					go func() {
						_, err := io.Copy(conn, fd)
						copyErr <- err
					}()

					// wait until one of the copy routines exit then
					// let the fd close
					err = <-copyErr
					if err != nil {
						log.Log.Object(vmi).Infof("closing notify pipe connection for vmi with error: %v", err)
					} else {
						log.Log.Object(vmi).Infof("gracefully closed notify pipe connection for vmi")
					}

				}(vmi)
			}
		}
	}(vmi, fdChan, domainPipeStopChan)
}

func (d *VirtualMachineController) startDomainNotifyPipe(domainPipeStopChan chan struct{}, vmi *v1.VirtualMachineInstance) error {

	res, err := d.podIsolationDetector.Detect(vmi)
	if err != nil {
		return fmt.Errorf("failed to detect isolation for launcher pod when setting up notify pipe: %v", err)
	}

	// inject the domain-notify.sock into the VMI pod.
	socketPath := filepath.Join(res.MountRoot(), d.virtShareDir, "domain-notify-pipe.sock")

	os.RemoveAll(socketPath)
	err = util.MkdirAllWithNosec(filepath.Dir(socketPath))
	if err != nil {
		log.Log.Reason(err).Error("unable to create directory for unix socket")
		return err
	}

	listener, err := net.Listen("unix", socketPath)
	if err != nil {
		log.Log.Reason(err).Error("failed to create unix socket for proxy service")
		return err
	}

	handleDomainNotifyPipe(domainPipeStopChan, listener, d.virtShareDir, vmi)

	return nil
}

// Determines if a domain's grace period has expired during shutdown.
// If the grace period has started but not expired, timeLeft represents
// the time in seconds left until the period expires.
// If the grace period has not started, timeLeft will be set to -1.
func (d *VirtualMachineController) hasGracePeriodExpired(dom *api.Domain) (hasExpired bool, timeLeft int64) {

	hasExpired = false
	timeLeft = 0

	if dom == nil {
		hasExpired = true
		return
	}

	startTime := int64(0)
	if dom.Spec.Metadata.KubeVirt.GracePeriod.DeletionTimestamp != nil {
		startTime = dom.Spec.Metadata.KubeVirt.GracePeriod.DeletionTimestamp.UTC().Unix()
	}
	gracePeriod := dom.Spec.Metadata.KubeVirt.GracePeriod.DeletionGracePeriodSeconds

	// If gracePeriod == 0, then there will be no startTime set, deletion
	// should occur immediately during shutdown.
	if gracePeriod == 0 {
		hasExpired = true
		return
	} else if startTime == 0 {
		// If gracePeriod > 0, then the shutdown signal needs to be sent
		// and the gracePeriod start time needs to be set.
		timeLeft = -1
		return
	}

	now := time.Now().UTC().Unix()
	diff := now - startTime

	if diff >= gracePeriod {
		hasExpired = true
		return
	}

	timeLeft = int64(gracePeriod - diff)
	if timeLeft < 1 {
		timeLeft = 1
	}
	return
}

func (d *VirtualMachineController) hasTargetDetectedDomain(vmi *v1.VirtualMachineInstance) (bool, int64) {
	// give the target node 60 seconds to discover the libvirt domain via the domain informer
	// before allowing the VMI to be processed. This closes the gap between the
	// VMI's status getting updated to reflect the new source node, and the domain
	// informer firing the event to alert the source node of the new domain.
	migrationTargetDelayTimeout := 60

	if vmi.Status.MigrationState != nil && vmi.Status.MigrationState.TargetNodeDomainDetected {

		return true, 0
	}

	nowUnix := time.Now().UTC().Unix()
	migrationEndUnix := vmi.Status.MigrationState.EndTimestamp.Time.UTC().Unix()

	diff := nowUnix - migrationEndUnix

	if diff > int64(migrationTargetDelayTimeout) {
		return false, 0
	}

	timeLeft := int64(migrationTargetDelayTimeout) - diff

	enqueueTime := timeLeft
	if enqueueTime < 5 {
		enqueueTime = 5
	}

	// re-enqueue the key to ensure it gets processed again within the right time.
	d.Queue.AddAfter(controller.VirtualMachineKey(vmi), time.Duration(enqueueTime)*time.Second)

	return false, timeLeft
}

func (d *VirtualMachineController) clearPodNetworkPhase1(vmi *v1.VirtualMachineInstance) {
	// no need to cleanup with empty uid
	if string(vmi.UID) == "" {
		return
	}
	d.phase1NetworkSetupCacheLock.Lock()
	delete(d.phase1NetworkSetupCache, vmi.UID)
	d.phase1NetworkSetupCacheLock.Unlock()

	// Clean Pod interface cache from map and files
	d.podInterfaceCacheLock.Lock()
	for key := range d.podInterfaceCache {
		if strings.Contains(key, string(vmi.UID)) {
			delete(d.podInterfaceCache, key)
		}
	}
	d.podInterfaceCacheLock.Unlock()

	err := d.networkCacheStoreFactory.CacheForVMI(vmi).Remove()
	if err != nil {
		log.Log.Reason(err).Errorf("failed to delete VMI Network cache files: %s", err.Error())
	}
}

// Reaching into the network namespace of the VMI's pod is expensive because
// it results in killing/spawning a posix thread. Only do this if it
// is absolutely necessary. The cache informs us if this action has
// already taken place or not for a VMI
func (d *VirtualMachineController) setPodNetworkPhase1(vmi *v1.VirtualMachineInstance) (bool, error) {

	// configure network
	res, err := d.podIsolationDetector.Detect(vmi)
	if err != nil {
		return false, fmt.Errorf("failed to detect isolation for launcher pod: %v", err)
	}

	pid := res.Pid()

	// check to see if we've already completed phase1 for this vmi
	d.phase1NetworkSetupCacheLock.Lock()
	cachedPid, ok := d.phase1NetworkSetupCache[vmi.UID]
	d.phase1NetworkSetupCacheLock.Unlock()

	if ok && cachedPid == pid {
		// already completed phase1
		return false, nil
	}

	err = res.DoNetNS(func() error {
		return network.NewVMNetworkConfigurator(vmi, d.networkCacheStoreFactory).SetupPodNetworkPhase1(pid)
	})
	if err != nil {
		_, critical := err.(*neterrors.CriticalNetworkError)
		if critical {
			return true, err
		} else {
			return false, err
		}

	}

	// cache that phase 1 has completed for this vmi.
	d.phase1NetworkSetupCacheLock.Lock()
	d.phase1NetworkSetupCache[vmi.UID] = pid
	d.phase1NetworkSetupCacheLock.Unlock()

	return false, nil
}

func domainMigrated(domain *api.Domain) bool {
	if domain != nil && domain.Status.Status == api.Shutoff && domain.Status.Reason == api.ReasonMigrated {
		return true
	}
	return false
}

func (d *VirtualMachineController) getPodInterfacefromFileCache(vmi *v1.VirtualMachineInstance, ifaceName string) (*netcache.PodCacheInterface, error) {
	cacheKey := fmt.Sprintf("%s/%s", vmi.UID, ifaceName)

	// Once the Interface files are set on the handler, they don't change
	// If already present in the map, don't read again
	d.podInterfaceCacheLock.Lock()
	result, exists := d.podInterfaceCache[cacheKey]
	d.podInterfaceCacheLock.Unlock()

	if exists {
		return result, nil
	}
	//FIXME error handling?
	result, _ = d.networkCacheStoreFactory.CacheForVMI(vmi).Read(ifaceName)

	d.podInterfaceCacheLock.Lock()
	d.podInterfaceCache[cacheKey] = result
	d.podInterfaceCacheLock.Unlock()

	return result, nil
}

func canUpdateToMounted(currentPhase v1.VolumePhase) bool {
	return currentPhase == v1.VolumeBound || currentPhase == v1.VolumePending || currentPhase == v1.HotplugVolumeAttachedToNode
}

func canUpdateToUnmounted(currentPhase v1.VolumePhase) bool {
	return currentPhase == v1.VolumeReady || currentPhase == v1.HotplugVolumeMounted || currentPhase == v1.HotplugVolumeAttachedToNode
}

func (d *VirtualMachineController) setMigrationProgressStatus(vmi *v1.VirtualMachineInstance, domain *api.Domain) *v1.VirtualMachineInstance {

	if domain == nil ||
		domain.Spec.Metadata.KubeVirt.Migration == nil ||
		vmi.Status.MigrationState == nil ||
		!d.isMigrationSource(vmi) {
		return vmi
	}

	migrationMetadata := domain.Spec.Metadata.KubeVirt.Migration
	if migrationMetadata.UID != vmi.Status.MigrationState.MigrationUID {
		return vmi
	}

	if vmi.Status.MigrationState.EndTimestamp == nil && migrationMetadata.EndTimestamp != nil {
		if migrationMetadata.Failed {
			d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Migrated.String(), fmt.Sprintf("VirtualMachineInstance migration uid %s failed. reason:%s", string(migrationMetadata.UID), migrationMetadata.FailureReason))
		}
	}

	if vmi.Status.MigrationState.StartTimestamp == nil {
		vmi.Status.MigrationState.StartTimestamp = migrationMetadata.StartTimestamp
	}
	if vmi.Status.MigrationState.EndTimestamp == nil {
		vmi.Status.MigrationState.EndTimestamp = migrationMetadata.EndTimestamp
	}
	vmi.Status.MigrationState.AbortStatus = v1.MigrationAbortStatus(migrationMetadata.AbortStatus)
	vmi.Status.MigrationState.Completed = migrationMetadata.Completed
	vmi.Status.MigrationState.Failed = migrationMetadata.Failed
	vmi.Status.MigrationState.Mode = migrationMetadata.Mode
	return vmi
}

func (d *VirtualMachineController) migrationSourceUpdateVMIStatus(origVMI *v1.VirtualMachineInstance, domain *api.Domain) error {

	vmi := origVMI.DeepCopy()
	oldStatus := vmi.DeepCopy().Status

	// if a migration happens very quickly, it's possible parts of the in
	// progress status wasn't set. We need to make sure we set this even
	// if the migration has completed
	vmi = d.setMigrationProgressStatus(vmi, domain)

	// handle migrations differently than normal status updates.
	//
	// When a successful migration is detected, we must transfer ownership of the VMI
	// from the source node (this node) to the target node (node the domain was migrated to).
	//
	// Transfer owership by...
	// 1. Marking vmi.Status.MigationState as completed
	// 2. Update the vmi.Status.NodeName to reflect the target node's name
	// 3. Update the VMI's NodeNameLabel annotation to reflect the target node's name
	// 4. Clear the LauncherContainerImageVersion which virt-controller will detect
	//    and accurately based on the version used on the target pod
	//
	// After a migration, the VMI's phase is no longer owned by this node. Only the
	// MigrationState status field is elgible to be mutated.
	migrationHost := ""
	if vmi.Status.MigrationState != nil {
		migrationHost = vmi.Status.MigrationState.TargetNode
	}

	if vmi.Status.MigrationState != nil && vmi.Status.MigrationState.EndTimestamp == nil {
		now := metav1.NewTime(time.Now())
		vmi.Status.MigrationState.EndTimestamp = &now
	}

	targetNodeDetectedDomain, timeLeft := d.hasTargetDetectedDomain(vmi)
	// If we can't detect where the migration went to, then we have no
	// way of transferring ownership. The only option here is to move the
	// vmi to failed.  The cluster vmi controller will then tear down the
	// resulting pods.
	if migrationHost == "" {
		// migrated to unknown host.
		vmi.Status.Phase = v1.Failed
		vmi.Status.MigrationState.Completed = true
		vmi.Status.MigrationState.Failed = true

		d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Migrated.String(), fmt.Sprintf("The VirtualMachineInstance migrated to unknown host."))
	} else if !targetNodeDetectedDomain {
		if timeLeft <= 0 {
			vmi.Status.Phase = v1.Failed
			vmi.Status.MigrationState.Completed = true
			vmi.Status.MigrationState.Failed = true

			d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Migrated.String(), fmt.Sprintf("The VirtualMachineInstance's domain was never observed on the target after the migration completed within the timeout period."))
		} else {
			log.Log.Object(vmi).Info("Waiting on the target node to observe the migrated domain before performing the handoff")
		}
	} else if vmi.Status.MigrationState != nil && vmi.Status.MigrationState.TargetNodeDomainDetected {
		// this is the migration ACK.
		// At this point we know that the migration has completed and that
		// the target node has seen the domain event.
		vmi.Labels[v1.NodeNameLabel] = migrationHost
		delete(vmi.Labels, v1.OutdatedLauncherImageLabel)
		vmi.Status.LauncherContainerImageVersion = ""
		vmi.Status.NodeName = migrationHost
		// clean the evacuation node name since have already migrated to a new node
		vmi.Status.EvacuationNodeName = ""
		vmi.Status.MigrationState.Completed = true
		d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.Migrated.String(), fmt.Sprintf("The VirtualMachineInstance migrated to node %s.", migrationHost))
		log.Log.Object(vmi).Infof("migration completed to node %s", migrationHost)
	}

	if !reflect.DeepEqual(oldStatus, vmi.Status) {
		_, err := d.clientset.VirtualMachineInstance(vmi.ObjectMeta.Namespace).Update(vmi)
		if err != nil {
			return err
		}
	}
	return nil
}

func (d *VirtualMachineController) migrationTargetUpdateVMIStatus(vmi *v1.VirtualMachineInstance, domainExists bool) error {

	vmiCopy := vmi.DeepCopy()

	if migrations.MigrationFailed(vmi) {
		// nothing left to report on the target node if the migration failed
		return nil
	}

	// Handle post migration
	if domainExists && vmi.Status.MigrationState != nil && !vmi.Status.MigrationState.TargetNodeDomainDetected {
		// record that we've see the domain populated on the target's node
		log.Log.Object(vmi).Info("The target node received the migrated domain")
		vmiCopy.Status.MigrationState.TargetNodeDomainDetected = true
		d.finalizeMigration(vmi)
	}

	if !migrations.IsMigrating(vmi) {
		destSrcPortsMap := d.migrationProxy.GetTargetListenerPorts(string(vmi.UID))
		if len(destSrcPortsMap) == 0 {
			msg := "target migration listener is not up for this vmi"
			log.Log.Object(vmi).Error(msg)
			return fmt.Errorf(msg)
		}

		hostAddress := ""

		// advertise the listener address to the source node
		if vmi.Status.MigrationState != nil {
			hostAddress = vmi.Status.MigrationState.TargetNodeAddress
		}
		if hostAddress != d.ipAddress {
			portsList := make([]string, 0, len(destSrcPortsMap))

			for k := range destSrcPortsMap {
				portsList = append(portsList, k)
			}
			portsStrList := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(portsList)), ","), "[]")
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.PreparingTarget.String(), fmt.Sprintf("Migration Target is listening at %s, on ports: %s", d.ipAddress, portsStrList))
			vmiCopy.Status.MigrationState.TargetNodeAddress = d.ipAddress
			vmiCopy.Status.MigrationState.TargetDirectMigrationNodePorts = destSrcPortsMap
		}
	}

	// update the VMI if necessary
	if !reflect.DeepEqual(vmi.Status, vmiCopy.Status) {
		_, err := d.clientset.VirtualMachineInstance(vmi.ObjectMeta.Namespace).Update(vmiCopy)
		if err != nil {
			return err
		}
	}

	return nil
}

func (d *VirtualMachineController) generateEventsForVolumeStatusChange(vmi *v1.VirtualMachineInstance, newStatusMap map[string]v1.VolumeStatus) {
	newStatusMapCopy := make(map[string]v1.VolumeStatus)
	for k, v := range newStatusMap {
		newStatusMapCopy[k] = v
	}
	for _, oldStatus := range vmi.Status.VolumeStatus {
		newStatus, ok := newStatusMap[oldStatus.Name]
		if !ok {
			// status got removed
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, VolumeUnplugged, fmt.Sprintf("Volume %s has been unplugged", oldStatus.Name))
			continue
		}
		if newStatus.Phase != oldStatus.Phase {
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, newStatus.Reason, newStatus.Message)
		}
		delete(newStatusMapCopy, newStatus.Name)
	}
	// Send events for any new statuses.
	for _, v := range newStatusMapCopy {
		d.recorder.Event(vmi, k8sv1.EventTypeNormal, v.Reason, v.Message)
	}
}

func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMachineInstance, volumeStatus v1.VolumeStatus, specVolumeMap map[string]v1.Volume) (v1.VolumeStatus, bool) {
	needsRefresh := false
	if volumeStatus.Target == "" {
		needsRefresh = true
		if mounted, _ := d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID); mounted {
			if _, ok := specVolumeMap[volumeStatus.Name]; ok && canUpdateToMounted(volumeStatus.Phase) {
				log.DefaultLogger().Infof("Marking volume %s as mounted in pod, it can now be attached", volumeStatus.Name)
				// mounted, and still in spec, and in phase we can change, update status to mounted.
				volumeStatus.Phase = v1.HotplugVolumeMounted
				volumeStatus.Message = fmt.Sprintf("Volume %s has been mounted in virt-launcher pod", volumeStatus.Name)
				volumeStatus.Reason = VolumeMountedToPodReason
			}
		} else {
			// Not mounted, check if the volume is in the spec, if not update status
			if _, ok := specVolumeMap[volumeStatus.Name]; !ok && canUpdateToUnmounted(volumeStatus.Phase) {
				log.DefaultLogger().Infof("Marking volume %s as unmounted from pod, it can now be detached", volumeStatus.Name)
				// Not mounted.
				volumeStatus.Phase = v1.HotplugVolumeUnMounted
				volumeStatus.Message = fmt.Sprintf("Volume %s has been unmounted from virt-launcher pod", volumeStatus.Name)
				volumeStatus.Reason = VolumeUnMountedFromPodReason
			}
		}
	} else {
		// Successfully attached to VM.
		volumeStatus.Phase = v1.VolumeReady
		volumeStatus.Message = fmt.Sprintf("Successfully attach hotplugged volume %s to VM", volumeStatus.Name)
		volumeStatus.Reason = VolumeReadyReason
	}
	return volumeStatus, needsRefresh
}

func (d *VirtualMachineController) updateVolumeStatusesFromDomain(vmi *v1.VirtualMachineInstance, domain *api.Domain) bool {
	hasHotplug := false
	if len(vmi.Status.VolumeStatus) > 0 {
		diskDeviceMap := make(map[string]string)
		for _, disk := range domain.Spec.Devices.Disks {
			diskDeviceMap[disk.Alias.GetName()] = disk.Target.Device
		}
		specVolumeMap := make(map[string]v1.Volume)
		for _, volume := range vmi.Spec.Volumes {
			specVolumeMap[volume.Name] = volume
		}
		newStatusMap := make(map[string]v1.VolumeStatus)
		newStatuses := make([]v1.VolumeStatus, 0)
		needsRefresh := false
		for _, volumeStatus := range vmi.Status.VolumeStatus {
			if _, ok := diskDeviceMap[volumeStatus.Name]; ok {
				volumeStatus.Target = diskDeviceMap[volumeStatus.Name]
			}
			if volumeStatus.HotplugVolume != nil {
				hasHotplug = true
				volumeStatus, needsRefresh = d.updateHotplugVolumeStatus(vmi, volumeStatus, specVolumeMap)
			}
			newStatuses = append(newStatuses, volumeStatus)
			newStatusMap[volumeStatus.Name] = volumeStatus
		}
		sort.SliceStable(newStatuses, func(i, j int) bool {
			return strings.Compare(newStatuses[i].Name, newStatuses[j].Name) == -1
		})
		if needsRefresh {
			d.Queue.AddAfter(controller.VirtualMachineKey(vmi), time.Second)
		}
		d.generateEventsForVolumeStatusChange(vmi, newStatusMap)
		vmi.Status.VolumeStatus = newStatuses
	}
	return hasHotplug
}

func (d *VirtualMachineController) updateVMIStatus(origVMI *v1.VirtualMachineInstance, domain *api.Domain, syncError error) (err error) {
	condManager := controller.NewVirtualMachineInstanceConditionManager()
	hasHotplug := false

	// Don't update the VirtualMachineInstance if it is already in a final state
	if origVMI.IsFinal() {
		return nil
	} else if origVMI.Status.NodeName != "" && origVMI.Status.NodeName != d.host {
		// Only update the VMI's phase if this node owns the VMI.
		// not owned by this host, likely the result of a migration
		return nil
	} else if domainMigrated(domain) {
		return d.migrationSourceUpdateVMIStatus(origVMI, domain)
	}

	vmi := origVMI.DeepCopy()
	oldStatus := vmi.DeepCopy().Status

	vmi = d.setMigrationProgressStatus(vmi, domain)

	if domain != nil {
		if vmi.Status.GuestOSInfo.Name != domain.Status.OSInfo.Name {
			vmi.Status.GuestOSInfo.Name = domain.Status.OSInfo.Name
			vmi.Status.GuestOSInfo.Version = domain.Status.OSInfo.VersionId
			vmi.Status.GuestOSInfo.KernelRelease = domain.Status.OSInfo.KernelRelease
			vmi.Status.GuestOSInfo.PrettyName = domain.Status.OSInfo.PrettyName
			vmi.Status.GuestOSInfo.VersionID = domain.Status.OSInfo.VersionId
			vmi.Status.GuestOSInfo.KernelVersion = domain.Status.OSInfo.KernelVersion
			vmi.Status.GuestOSInfo.ID = domain.Status.OSInfo.Id
		}
		// This is needed to be backwards compatible with vmi's which have status interfaces
		// with the name not being set
		if len(domain.Spec.Devices.Interfaces) == 0 && len(vmi.Status.Interfaces) == 1 && vmi.Status.Interfaces[0].Name == "" {
			for _, network := range vmi.Spec.Networks {
				if network.NetworkSource.Pod != nil {
					vmi.Status.Interfaces[0].Name = network.Name
				}
			}
		}

		hasHotplug = d.updateVolumeStatusesFromDomain(vmi, domain)
		if len(vmi.Status.Interfaces) == 0 {
			// Set Pod Interface
			interfaces := make([]v1.VirtualMachineInstanceNetworkInterface, 0)
			for _, network := range vmi.Spec.Networks {
				if network.NetworkSource.Pod != nil {
					podIface, err := d.getPodInterfacefromFileCache(vmi, network.Name)
					if err != nil {
						return err
					}
					ifc := v1.VirtualMachineInstanceNetworkInterface{
						Name: network.Name,
						IP:   podIface.PodIP,
						IPs:  podIface.PodIPs,
					}
					interfaces = append(interfaces, ifc)
				}
			}
			vmi.Status.Interfaces = interfaces
		}

		if len(domain.Spec.Devices.Interfaces) > 0 || len(domain.Status.Interfaces) > 0 {
			// This calculates the vmi.Status.Interfaces based on the following data sets:
			// - vmi.Status.Interfaces - previously calculated interfaces, this can contain data (pod IP)
			//   set in the previous loops (when there are no interfaces), which can not be deleted,
			//   unless overridden by Qemu agent
			// - domain.Spec - interfaces form the Spec
			// - domain.Status.Interfaces - interfaces reported by guest agent (empty if Qemu agent not running)
			newInterfaces := []v1.VirtualMachineInstanceNetworkInterface{}

			existingInterfaceStatusByName := map[string]v1.VirtualMachineInstanceNetworkInterface{}
			for _, existingInterfaceStatus := range vmi.Status.Interfaces {
				if existingInterfaceStatus.Name != "" {
					existingInterfaceStatusByName[existingInterfaceStatus.Name] = existingInterfaceStatus
				}
			}

			domainInterfaceStatusByMac := map[string]api.InterfaceStatus{}
			for _, domainInterfaceStatus := range domain.Status.Interfaces {
				domainInterfaceStatusByMac[domainInterfaceStatus.Mac] = domainInterfaceStatus
			}

			existingInterfacesSpecByName := map[string]v1.Interface{}
			for _, existingInterfaceSpec := range vmi.Spec.Domain.Devices.Interfaces {
				existingInterfacesSpecByName[existingInterfaceSpec.Name] = existingInterfaceSpec
			}
			existingNetworksByName := map[string]v1.Network{}
			for _, existingNetwork := range vmi.Spec.Networks {
				existingNetworksByName[existingNetwork.Name] = existingNetwork
			}

			// Iterate through all domain.Spec interfaces
			for _, domainInterface := range domain.Spec.Devices.Interfaces {
				interfaceMAC := domainInterface.MAC.MAC
				var newInterface v1.VirtualMachineInstanceNetworkInterface
				var isForwardingBindingInterface = false

				if existingInterfacesSpecByName[domainInterface.Alias.GetName()].Masquerade != nil || existingInterfacesSpecByName[domainInterface.Alias.GetName()].Slirp != nil {
					isForwardingBindingInterface = true
				}

				if existingInterface, exists := existingInterfaceStatusByName[domainInterface.Alias.GetName()]; exists {
					// Reuse previously calculated interface from vmi.Status.Interfaces, updating the MAC from domain.Spec
					// Only interfaces defined in domain.Spec are handled here
					newInterface = existingInterface
					newInterface.MAC = interfaceMAC

					// If it is a Combination of Masquerade+Pod network, check IP from file cache
					if existingInterfacesSpecByName[domainInterface.Alias.GetName()].Masquerade != nil && existingNetworksByName[domainInterface.Alias.GetName()].NetworkSource.Pod != nil {
						iface, err := d.getPodInterfacefromFileCache(vmi, domainInterface.Alias.GetName())
						if err != nil {
							return err
						}

						if !reflect.DeepEqual(iface.PodIPs, existingInterfaceStatusByName[domainInterface.Alias.GetName()].IPs) {
							newInterface.Name = domainInterface.Alias.GetName()
							newInterface.IP = iface.PodIP
							newInterface.IPs = iface.PodIPs
						}
					}
				} else {
					// If not present in vmi.Status.Interfaces, create a new one based on domain.Spec
					newInterface = v1.VirtualMachineInstanceNetworkInterface{
						MAC:  interfaceMAC,
						Name: domainInterface.Alias.GetName(),
					}
				}

				// Update IP info based on information from domain.Status.Interfaces (Qemu guest)
				// Remove the interface from domainInterfaceStatusByMac to mark it as handled
				if interfaceStatus, exists := domainInterfaceStatusByMac[interfaceMAC]; exists {
					newInterface.InterfaceName = interfaceStatus.InterfaceName
					// Do not update if interface has Masquerede binding
					// virt-controller should update VMI status interface with Pod IP instead
					if !isForwardingBindingInterface {
						newInterface.IP = interfaceStatus.Ip
						newInterface.IPs = interfaceStatus.IPs
					}
					delete(domainInterfaceStatusByMac, interfaceMAC)
				}
				newInterfaces = append(newInterfaces, newInterface)
			}

			// If any of domain.Status.Interfaces were not handled above, it means that the vm contains additional
			// interfaces not defined in domain.Spec.Devices.Interfaces (most likely added by user on VM or a SRIOV interface)
			// Add them to vmi.Status.Interfaces
			setMissingSRIOVInterfacesNames(existingInterfacesSpecByName, domainInterfaceStatusByMac)
			for interfaceMAC, domainInterfaceStatus := range domainInterfaceStatusByMac {
				newInterface := v1.VirtualMachineInstanceNetworkInterface{
					Name:          domainInterfaceStatus.Name,
					MAC:           interfaceMAC,
					IP:            domainInterfaceStatus.Ip,
					IPs:           domainInterfaceStatus.IPs,
					InterfaceName: domainInterfaceStatus.InterfaceName,
				}
				newInterfaces = append(newInterfaces, newInterface)
			}
			vmi.Status.Interfaces = newInterfaces
		}
	}

	// Update AccessCredential conditions
	if domain != nil && domain.Spec.Metadata.KubeVirt.AccessCredential != nil {

		message := domain.Spec.Metadata.KubeVirt.AccessCredential.Message
		status := k8sv1.ConditionFalse
		if domain.Spec.Metadata.KubeVirt.AccessCredential.Succeeded {
			status = k8sv1.ConditionTrue
		}

		add := false
		condition := condManager.GetCondition(vmi, v1.VirtualMachineInstanceAccessCredentialsSynchronized)
		if condition == nil {
			add = true
		} else if condition.Status != status || condition.Message != message {
			// if not as expected, remove, then add.
			condManager.RemoveCondition(vmi, v1.VirtualMachineInstanceAccessCredentialsSynchronized)
			add = true
		}
		if add {
			newCondition := v1.VirtualMachineInstanceCondition{
				Type:               v1.VirtualMachineInstanceAccessCredentialsSynchronized,
				LastTransitionTime: metav1.Now(),
				Status:             status,
				Message:            message,
			}
			vmi.Status.Conditions = append(vmi.Status.Conditions, newCondition)
			if status == k8sv1.ConditionTrue {
				d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.AccessCredentialsSyncSuccess.String(), message)
			} else {
				d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.AccessCredentialsSyncFailed.String(), message)
			}
		}
	}

	// Calculate the new VirtualMachineInstance state based on what libvirt reported
	err = d.setVmPhaseForStatusReason(domain, vmi)
	if err != nil {
		return err
	}

	// Cacluate whether the VM is migratable
	liveMigrationCondition, isBlockMigration := d.calculateLiveMigrationCondition(vmi, hasHotplug)
	if !condManager.HasCondition(vmi, v1.VirtualMachineInstanceIsMigratable) {
		vmi.Status.Conditions = append(vmi.Status.Conditions, *liveMigrationCondition)
		// Set VMI Migration Method
		if isBlockMigration {
			vmi.Status.MigrationMethod = v1.BlockMigration
		} else {
			vmi.Status.MigrationMethod = v1.LiveMigration
		}
	} else {
		cond := condManager.GetCondition(vmi, v1.VirtualMachineInstanceIsMigratable)
		if !reflect.DeepEqual(cond, liveMigrationCondition) {
			condManager.RemoveCondition(vmi, v1.VirtualMachineInstanceIsMigratable)
			vmi.Status.Conditions = append(vmi.Status.Conditions, *liveMigrationCondition)
		}
	}
	if vmi.IsEvictable() && liveMigrationCondition.Status == k8sv1.ConditionFalse {
		d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Migrated.String(), "EvictionStrategy is set but vmi is not migratable")
	}
	// Update the condition when GA is connected
	channelConnected := false
	if domain != nil {
		for _, channel := range domain.Spec.Devices.Channels {
			if channel.Target != nil {
				log.Log.V(4).Infof("Channel: %s, %s", channel.Target.Name, channel.Target.State)
				if channel.Target.Name == "org.qemu.guest_agent.0" {
					if channel.Target.State == "connected" {
						channelConnected = true
					}
				}

			}
		}
	}

	switch {
	case channelConnected && !condManager.HasCondition(vmi, v1.VirtualMachineInstanceAgentConnected):
		agentCondition := v1.VirtualMachineInstanceCondition{
			Type:          v1.VirtualMachineInstanceAgentConnected,
			LastProbeTime: metav1.Now(),
			Status:        k8sv1.ConditionTrue,
		}
		vmi.Status.Conditions = append(vmi.Status.Conditions, agentCondition)
	case !channelConnected:
		condManager.RemoveCondition(vmi, v1.VirtualMachineInstanceAgentConnected)
	}

	if condManager.HasCondition(vmi, v1.VirtualMachineInstanceAgentConnected) {
		client, err := d.getLauncherClient(vmi)
		if err != nil {
			return err
		}

		guestInfo, err := client.GetGuestInfo()
		if err != nil {
			return err
		}

		var supported = false

		// For current versions, virt-launcher's supported commands will always contain data.
		// For backwards compatibility: during upgrade from a previous version of KubeVirt,
		// virt-launcher might not provide any supported commands. If the list of supported
		// commands is empty, fall back to previous behavior.
		if len(guestInfo.SupportedCommands) > 0 {
			supported = isGuestAgentSupported(vmi, guestInfo.SupportedCommands)
		} else {
			for _, version := range d.clusterConfig.GetSupportedAgentVersions() {
				supported = supported || regexp.MustCompile(version).MatchString(guestInfo.GAVersion)
			}
		}

		if !supported {
			if !condManager.HasCondition(vmi, v1.VirtualMachineInstanceUnsupportedAgent) {
				agentCondition := v1.VirtualMachineInstanceCondition{
					Type:          v1.VirtualMachineInstanceUnsupportedAgent,
					LastProbeTime: metav1.Now(),
					Status:        k8sv1.ConditionTrue,
				}
				vmi.Status.Conditions = append(vmi.Status.Conditions, agentCondition)
			}
		} else {
			condManager.RemoveCondition(vmi, v1.VirtualMachineInstanceUnsupportedAgent)
		}

	}

	// Update paused condition in case VMI was paused / unpaused
	if domain != nil && domain.Status.Status == api.Paused && domain.Status.Reason == api.ReasonPausedUser {
		if !condManager.HasCondition(vmi, v1.VirtualMachineInstancePaused) {
			log.Log.Object(vmi).V(3).Info("Adding paused condition")
			now := metav1.NewTime(time.Now())
			vmi.Status.Conditions = append(vmi.Status.Conditions, v1.VirtualMachineInstanceCondition{
				Type:               v1.VirtualMachineInstancePaused,
				Status:             k8sv1.ConditionTrue,
				LastProbeTime:      now,
				LastTransitionTime: now,
				Reason:             "PausedByUser",
				Message:            "VMI was paused by user",
			})
		}
	} else if condManager.HasCondition(vmi, v1.VirtualMachineInstancePaused) {
		log.Log.Object(vmi).V(3).Info("Removing paused condition")
		condManager.RemoveCondition(vmi, v1.VirtualMachineInstancePaused)
	}

	if _, ok := syncError.(*virtLauncherCriticalNetworkError); ok {
		log.Log.Errorf("virt-launcher crashed due to a network error. Updating VMI %s status to Failed", vmi.Name)
		vmi.Status.Phase = v1.Failed
	}
	if _, ok := syncError.(*virtLauncherCriticalSecurebootError); ok {
		log.Log.Errorf("virt-launcher does not support the Secure Boot setting. Updating VMI %s status to Failed", vmi.Name)
		vmi.Status.Phase = v1.Failed
	}
	condManager.CheckFailure(vmi, syncError, "Synchronizing with the Domain failed.")

	if !reflect.DeepEqual(oldStatus, vmi.Status) {
		_, err = d.clientset.VirtualMachineInstance(vmi.ObjectMeta.Namespace).Update(vmi)
		if err != nil {
			return err
		}
	}

	if oldStatus.Phase != vmi.Status.Phase {
		switch vmi.Status.Phase {
		case v1.Running:
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.Started.String(), VMIStarted)
		case v1.Succeeded:
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.Stopped.String(), VMIShutdown)
		case v1.Failed:
			d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Stopped.String(), VMICrashed)
		}
	}

	return nil
}

func _guestAgentCommandSubsetSupported(requiredCommands []string, commands []v1.GuestAgentCommandInfo) bool {
	var found bool
	for _, cmd := range requiredCommands {
		found = false
		for _, foundCmd := range commands {
			if cmd == foundCmd.Name {
				if foundCmd.Enabled {
					found = true
				}
				break
			}
		}
		if found == false {
			return false
		}
	}
	return true

}

func isGuestAgentSupported(vmi *v1.VirtualMachineInstance, commands []v1.GuestAgentCommandInfo) bool {
	log.Log.V(3).Object(vmi).Infof("checking guest agent: %v", commands)
	if !_guestAgentCommandSubsetSupported(RequiredGuestAgentCommands, commands) {
		log.Log.V(3).Object(vmi).Info("This guest agent doesn't support required basic commands")
		return false
	}

	checkSSH := false
	checkPasswd := false

	if vmi != nil && vmi.Spec.AccessCredentials != nil {
		for _, accessCredential := range vmi.Spec.AccessCredentials {
			if accessCredential.SSHPublicKey != nil && accessCredential.SSHPublicKey.PropagationMethod.QemuGuestAgent != nil {
				// defer checking the command list so we only do that once
				checkSSH = true
			}
			if accessCredential.UserPassword != nil && accessCredential.UserPassword.PropagationMethod.QemuGuestAgent != nil {
				// defer checking the command list so we only do that once
				checkPasswd = true
			}

		}
	}

	if checkSSH && !_guestAgentCommandSubsetSupported(SSHRelatedGuestAgentCommands, commands) {
		log.Log.V(3).Object(vmi).Info("This guest agent doesn't support required public key commands")
		return false
	}

	if checkPasswd && !_guestAgentCommandSubsetSupported(PasswordRelatedGuestAgentCommands, commands) {
		log.Log.V(3).Object(vmi).Info("This guest agent doesn't support required password commands")
		return false
	}

	log.Log.V(3).Object(vmi).Info("This guest agent is supported")
	return true
}<commit_msg>Move adding of pause condition to separate func<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package virthandler

import (
	"context"
	goerror "errors"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/opencontainers/runc/libcontainer/cgroups"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	netcache "kubevirt.io/kubevirt/pkg/network/cache"
	"kubevirt.io/kubevirt/pkg/util"

	"kubevirt.io/kubevirt/pkg/virt-handler/heartbeat"

	"kubevirt.io/kubevirt/pkg/util/migrations"

	container_disk "kubevirt.io/kubevirt/pkg/virt-handler/container-disk"
	device_manager "kubevirt.io/kubevirt/pkg/virt-handler/device-manager"
	hotplug_volume "kubevirt.io/kubevirt/pkg/virt-handler/hotplug-disk"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/client-go/log"
	"kubevirt.io/kubevirt/pkg/controller"
	diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils"
	cmdv1 "kubevirt.io/kubevirt/pkg/handler-launcher-com/cmd/v1"
	hostdisk "kubevirt.io/kubevirt/pkg/host-disk"
	neterrors "kubevirt.io/kubevirt/pkg/network/errors"
	pvcutils "kubevirt.io/kubevirt/pkg/util/types"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
	virtcache "kubevirt.io/kubevirt/pkg/virt-handler/cache"
	cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
	"kubevirt.io/kubevirt/pkg/virt-handler/isolation"
	migrationproxy "kubevirt.io/kubevirt/pkg/virt-handler/migration-proxy"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/network"
	"kubevirt.io/kubevirt/pkg/watchdog"
)

const (
	//VolumeReadyReason is the reason set when the volume is ready.
	VolumeReadyReason = "VolumeReady"
	//VolumeUnMountedFromPodReason is the reason set when the volume is unmounted from the virtlauncher pod
	VolumeUnMountedFromPodReason = "VolumeUnMountedFromPod"
	//VolumeMountedToPodReason is the reason set when the volume is mounted to the virtlauncher pod
	VolumeMountedToPodReason = "VolumeMountedToPod"
	//VolumeUnplugged is the reason set when the volume is completely unplugged from the VMI
	VolumeUnplugged = "VolumeUnplugged"
	//VMIDefined is the reason set when a VMI is defined
	VMIDefined = "VirtualMachineInstance defined."
	//VMIStarted is the reason set when a VMI is started
	VMIStarted = "VirtualMachineInstance started."
	//VMIShutdown is the reason set when a VMI is shutdown
	VMIShutdown = "The VirtualMachineInstance was shut down."
	//VMICrashed is the reason set when a VMI crashed
	VMICrashed = "The VirtualMachineInstance crashed."
	//VMIAbortingMigration is the reason set when migration is being aborted
	VMIAbortingMigration = "VirtualMachineInstance is aborting migration."
	//VMIMigrating in the reason set when the VMI is migrating
	VMIMigrating = "VirtualMachineInstance is migrating."
	//VMIMigrationTargetPrepared is the reason set when the migration target has been prepared
	VMIMigrationTargetPrepared = "VirtualMachineInstance Migration Target Prepared."
	//VMIStopping is the reason set when the VMI is stopping
	VMIStopping = "VirtualMachineInstance stopping"
	//VMIGracefulShutdown is the reason set when the VMI is gracefully shut down
	VMIGracefulShutdown = "Signaled Graceful Shutdown"
	//VMISignalDeletion is the reason set when the VMI has signal deletion
	VMISignalDeletion = "Signaled Deletion"
)

var RequiredGuestAgentCommands = []string{
	"guest-ping",
	"guest-get-time",
	"guest-info",
	"guest-shutdown",
	"guest-network-get-interfaces",
	"guest-get-fsinfo",
	"guest-get-host-name",
	"guest-get-users",
	"guest-get-timezone",
	"guest-get-osinfo",
}

var SSHRelatedGuestAgentCommands = []string{
	"guest-exec-status",
	"guest-exec",
	"guest-file-open",
	"guest-file-close",
	"guest-file-read",
	"guest-file-write",
}

var PasswordRelatedGuestAgentCommands = []string{
	"guest-set-user-password",
}

type launcherClientInfo struct {
	client              cmdclient.LauncherClient
	socketFile          string
	domainPipeStopChan  chan struct{}
	notInitializedSince time.Time
	ready               bool
}

func NewController(
	recorder record.EventRecorder,
	clientset kubecli.KubevirtClient,
	host string,
	ipAddress string,
	virtShareDir string,
	virtPrivateDir string,
	vmiSourceInformer cache.SharedIndexInformer,
	vmiTargetInformer cache.SharedIndexInformer,
	domainInformer cache.SharedInformer,
	gracefulShutdownInformer cache.SharedIndexInformer,
	watchdogTimeoutSeconds int,
	maxDevices int,
	clusterConfig *virtconfig.ClusterConfig,
	podIsolationDetector isolation.PodIsolationDetector,
	migrationProxy migrationproxy.ProxyManager,
) *VirtualMachineController {

	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

	c := &VirtualMachineController{
		Queue:                       queue,
		recorder:                    recorder,
		clientset:                   clientset,
		host:                        host,
		ipAddress:                   ipAddress,
		virtShareDir:                virtShareDir,
		vmiSourceInformer:           vmiSourceInformer,
		vmiTargetInformer:           vmiTargetInformer,
		domainInformer:              domainInformer,
		gracefulShutdownInformer:    gracefulShutdownInformer,
		heartBeatInterval:           1 * time.Minute,
		watchdogTimeoutSeconds:      watchdogTimeoutSeconds,
		migrationProxy:              migrationProxy,
		podIsolationDetector:        podIsolationDetector,
		containerDiskMounter:        container_disk.NewMounter(podIsolationDetector, virtPrivateDir+"/container-disk-mount-state"),
		hotplugVolumeMounter:        hotplug_volume.NewVolumeMounter(podIsolationDetector, virtPrivateDir+"/hotplug-volume-mount-state"),
		clusterConfig:               clusterConfig,
		networkCacheStoreFactory:    netcache.NewInterfaceCacheFactory(),
		virtLauncherFSRunDirPattern: "/proc/%d/root/var/run",
	}

	vmiSourceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addFunc,
		DeleteFunc: c.deleteFunc,
		UpdateFunc: c.updateFunc,
	})

	vmiTargetInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addFunc,
		DeleteFunc: c.deleteFunc,
		UpdateFunc: c.updateFunc,
	})

	domainInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addDomainFunc,
		DeleteFunc: c.deleteDomainFunc,
		UpdateFunc: c.updateDomainFunc,
	})

	gracefulShutdownInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addFunc,
		DeleteFunc: c.deleteFunc,
		UpdateFunc: c.updateFunc,
	})

	c.launcherClients = make(map[types.UID]*launcherClientInfo)
	c.phase1NetworkSetupCache = make(map[types.UID]int)
	c.podInterfaceCache = make(map[string]*netcache.PodCacheInterface)

	c.domainNotifyPipes = make(map[string]string)

	permissions := "rw"
	if cgroups.IsCgroup2UnifiedMode() {
		// Need 'rwm' permissions otherwise ebpf filtering program attached by runc
		// will deny probing the device file with 'access' syscall. That in turn
		// will lead to libvirtd failure on VM startup.
		// This has been fixed upstream:
		//   https://github.com/opencontainers/runc/pull/2796
		// but the workaround is still needed to support previous versions without
		// the patch.
		permissions = "rwm"
	}

	c.deviceManagerController = device_manager.NewDeviceController(c.host, maxDevices, permissions, clusterConfig)
	c.heartBeat = heartbeat.NewHeartBeat(clientset.CoreV1(), c.deviceManagerController, clusterConfig, host)

	return c
}

type VirtualMachineController struct {
	recorder                 record.EventRecorder
	clientset                kubecli.KubevirtClient
	host                     string
	ipAddress                string
	virtShareDir             string
	virtPrivateDir           string
	Queue                    workqueue.RateLimitingInterface
	vmiSourceInformer        cache.SharedIndexInformer
	vmiTargetInformer        cache.SharedIndexInformer
	domainInformer           cache.SharedInformer
	gracefulShutdownInformer cache.SharedIndexInformer
	launcherClients          map[types.UID]*launcherClientInfo
	launcherClientLock       sync.Mutex
	heartBeatInterval        time.Duration
	watchdogTimeoutSeconds   int
	deviceManagerController  *device_manager.DeviceController
	migrationProxy           migrationproxy.ProxyManager
	podIsolationDetector     isolation.PodIsolationDetector
	containerDiskMounter     container_disk.Mounter
	hotplugVolumeMounter     hotplug_volume.VolumeMounter
	clusterConfig            *virtconfig.ClusterConfig

	// records if pod network phase1 has completed
	// phase1 involves cycling an entire posix thread
	// so for performance, knowing phase1 is complete
	// prevents cycling an unncessary posix thread.
	phase1NetworkSetupCache     map[types.UID]int
	phase1NetworkSetupCacheLock sync.Mutex

	// key is the file path, value is the contents.
	// if key exists, then don't read directly from file.
	podInterfaceCache     map[string]*netcache.PodCacheInterface
	podInterfaceCacheLock sync.Mutex

	domainNotifyPipes           map[string]string
	networkCacheStoreFactory    netcache.InterfaceCacheFactory
	virtLauncherFSRunDirPattern string
	heartBeat                   *heartbeat.HeartBeat
}

type virtLauncherCriticalNetworkError struct {
	msg string
}

func (e *virtLauncherCriticalNetworkError) Error() string { return e.msg }

type virtLauncherCriticalSecurebootError struct {
	msg string
}

func (e *virtLauncherCriticalSecurebootError) Error() string { return e.msg }

func handleDomainNotifyPipe(domainPipeStopChan chan struct{}, ln net.Listener, virtShareDir string, vmi *v1.VirtualMachineInstance) {

	fdChan := make(chan net.Conn, 100)

	// Listen for new connections,
	// Close listener and exit when stop encountered
	go func(vmi *v1.VirtualMachineInstance, ln net.Listener, domainPipeStopChan chan struct{}) {
		for {
			select {
			case <-domainPipeStopChan:
				log.Log.Object(vmi).Infof("closing notify pipe listener for vmi")
				ln.Close()
				return
			default:
				fd, err := ln.Accept()
				if err != nil {
					log.Log.Reason(err).Error("Domain pipe accept error encountered.")
					// keep listening until stop invoked
					time.Sleep(1)
				} else {
					fdChan <- fd
				}
			}
		}
	}(vmi, ln, domainPipeStopChan)

	// Process new connections
	// exit when stop encountered
	go func(vmi *v1.VirtualMachineInstance, fdChan chan net.Conn, domainPipeStopChan chan struct{}) {
		for {
			select {
			case <-domainPipeStopChan:
				return
			case fd := <-fdChan:
				go func(vmi *v1.VirtualMachineInstance) {
					defer fd.Close()

					// pipe the VMI domain-notify.sock to the virt-handler domain-notify.sock
					// so virt-handler receives notifications from the VMI
					conn, err := net.Dial("unix", filepath.Join(virtShareDir, "domain-notify.sock"))
					if err != nil {
						log.Log.Reason(err).Error("error connecting to domain-notify.sock for proxy connection")
						return
					}
					defer conn.Close()

					log.Log.Object(vmi).Infof("Accepted new notify pipe connection for vmi")
					copyErr := make(chan error, 2)
					go func() {
						_, err := io.Copy(fd, conn)
						copyErr <- err
					}()
					go func() {
						_, err := io.Copy(conn, fd)
						copyErr <- err
					}()

					// wait until one of the copy routines exit then
					// let the fd close
					err = <-copyErr
					if err != nil {
						log.Log.Object(vmi).Infof("closing notify pipe connection for vmi with error: %v", err)
					} else {
						log.Log.Object(vmi).Infof("gracefully closed notify pipe connection for vmi")
					}

				}(vmi)
			}
		}
	}(vmi, fdChan, domainPipeStopChan)
}

func (d *VirtualMachineController) startDomainNotifyPipe(domainPipeStopChan chan struct{}, vmi *v1.VirtualMachineInstance) error {

	res, err := d.podIsolationDetector.Detect(vmi)
	if err != nil {
		return fmt.Errorf("failed to detect isolation for launcher pod when setting up notify pipe: %v", err)
	}

	// inject the domain-notify.sock into the VMI pod.
	socketPath := filepath.Join(res.MountRoot(), d.virtShareDir, "domain-notify-pipe.sock")

	os.RemoveAll(socketPath)
	err = util.MkdirAllWithNosec(filepath.Dir(socketPath))
	if err != nil {
		log.Log.Reason(err).Error("unable to create directory for unix socket")
		return err
	}

	listener, err := net.Listen("unix", socketPath)
	if err != nil {
		log.Log.Reason(err).Error("failed to create unix socket for proxy service")
		return err
	}

	handleDomainNotifyPipe(domainPipeStopChan, listener, d.virtShareDir, vmi)

	return nil
}

// Determines if a domain's grace period has expired during shutdown.
// If the grace period has started but not expired, timeLeft represents
// the time in seconds left until the period expires.
// If the grace period has not started, timeLeft will be set to -1.
func (d *VirtualMachineController) hasGracePeriodExpired(dom *api.Domain) (hasExpired bool, timeLeft int64) {

	hasExpired = false
	timeLeft = 0

	if dom == nil {
		hasExpired = true
		return
	}

	startTime := int64(0)
	if dom.Spec.Metadata.KubeVirt.GracePeriod.DeletionTimestamp != nil {
		startTime = dom.Spec.Metadata.KubeVirt.GracePeriod.DeletionTimestamp.UTC().Unix()
	}
	gracePeriod := dom.Spec.Metadata.KubeVirt.GracePeriod.DeletionGracePeriodSeconds

	// If gracePeriod == 0, then there will be no startTime set, deletion
	// should occur immediately during shutdown.
	if gracePeriod == 0 {
		hasExpired = true
		return
	} else if startTime == 0 {
		// If gracePeriod > 0, then the shutdown signal needs to be sent
		// and the gracePeriod start time needs to be set.
		timeLeft = -1
		return
	}

	now := time.Now().UTC().Unix()
	diff := now - startTime

	if diff >= gracePeriod {
		hasExpired = true
		return
	}

	timeLeft = int64(gracePeriod - diff)
	if timeLeft < 1 {
		timeLeft = 1
	}
	return
}

func (d *VirtualMachineController) hasTargetDetectedDomain(vmi *v1.VirtualMachineInstance) (bool, int64) {
	// give the target node 60 seconds to discover the libvirt domain via the domain informer
	// before allowing the VMI to be processed. This closes the gap between the
	// VMI's status getting updated to reflect the new source node, and the domain
	// informer firing the event to alert the source node of the new domain.
	migrationTargetDelayTimeout := 60

	if vmi.Status.MigrationState != nil && vmi.Status.MigrationState.TargetNodeDomainDetected {

		return true, 0
	}

	nowUnix := time.Now().UTC().Unix()
	migrationEndUnix := vmi.Status.MigrationState.EndTimestamp.Time.UTC().Unix()

	diff := nowUnix - migrationEndUnix

	if diff > int64(migrationTargetDelayTimeout) {
		return false, 0
	}

	timeLeft := int64(migrationTargetDelayTimeout) - diff

	enqueueTime := timeLeft
	if enqueueTime < 5 {
		enqueueTime = 5
	}

	// re-enqueue the key to ensure it gets processed again within the right time.
	d.Queue.AddAfter(controller.VirtualMachineKey(vmi), time.Duration(enqueueTime)*time.Second)

	return false, timeLeft
}

func (d *VirtualMachineController) clearPodNetworkPhase1(vmi *v1.VirtualMachineInstance) {
	// no need to cleanup with empty uid
	if string(vmi.UID) == "" {
		return
	}
	d.phase1NetworkSetupCacheLock.Lock()
	delete(d.phase1NetworkSetupCache, vmi.UID)
	d.phase1NetworkSetupCacheLock.Unlock()

	// Clean Pod interface cache from map and files
	d.podInterfaceCacheLock.Lock()
	for key := range d.podInterfaceCache {
		if strings.Contains(key, string(vmi.UID)) {
			delete(d.podInterfaceCache, key)
		}
	}
	d.podInterfaceCacheLock.Unlock()

	err := d.networkCacheStoreFactory.CacheForVMI(vmi).Remove()
	if err != nil {
		log.Log.Reason(err).Errorf("failed to delete VMI Network cache files: %s", err.Error())
	}
}

// Reaching into the network namespace of the VMI's pod is expensive because
// it results in killing/spawning a posix thread. Only do this if it
// is absolutely necessary. The cache informs us if this action has
// already taken place or not for a VMI
func (d *VirtualMachineController) setPodNetworkPhase1(vmi *v1.VirtualMachineInstance) (bool, error) {

	// configure network
	res, err := d.podIsolationDetector.Detect(vmi)
	if err != nil {
		return false, fmt.Errorf("failed to detect isolation for launcher pod: %v", err)
	}

	pid := res.Pid()

	// check to see if we've already completed phase1 for this vmi
	d.phase1NetworkSetupCacheLock.Lock()
	cachedPid, ok := d.phase1NetworkSetupCache[vmi.UID]
	d.phase1NetworkSetupCacheLock.Unlock()

	if ok && cachedPid == pid {
		// already completed phase1
		return false, nil
	}

	err = res.DoNetNS(func() error {
		return network.NewVMNetworkConfigurator(vmi, d.networkCacheStoreFactory).SetupPodNetworkPhase1(pid)
	})
	if err != nil {
		_, critical := err.(*neterrors.CriticalNetworkError)
		if critical {
			return true, err
		} else {
			return false, err
		}

	}

	// cache that phase 1 has completed for this vmi.
	d.phase1NetworkSetupCacheLock.Lock()
	d.phase1NetworkSetupCache[vmi.UID] = pid
	d.phase1NetworkSetupCacheLock.Unlock()

	return false, nil
}

func domainMigrated(domain *api.Domain) bool {
	if domain != nil && domain.Status.Status == api.Shutoff && domain.Status.Reason == api.ReasonMigrated {
		return true
	}
	return false
}

func (d *VirtualMachineController) getPodInterfacefromFileCache(vmi *v1.VirtualMachineInstance, ifaceName string) (*netcache.PodCacheInterface, error) {
	cacheKey := fmt.Sprintf("%s/%s", vmi.UID, ifaceName)

	// Once the Interface files are set on the handler, they don't change
	// If already present in the map, don't read again
	d.podInterfaceCacheLock.Lock()
	result, exists := d.podInterfaceCache[cacheKey]
	d.podInterfaceCacheLock.Unlock()

	if exists {
		return result, nil
	}
	//FIXME error handling?
	result, _ = d.networkCacheStoreFactory.CacheForVMI(vmi).Read(ifaceName)

	d.podInterfaceCacheLock.Lock()
	d.podInterfaceCache[cacheKey] = result
	d.podInterfaceCacheLock.Unlock()

	return result, nil
}

func canUpdateToMounted(currentPhase v1.VolumePhase) bool {
	return currentPhase == v1.VolumeBound || currentPhase == v1.VolumePending || currentPhase == v1.HotplugVolumeAttachedToNode
}

func canUpdateToUnmounted(currentPhase v1.VolumePhase) bool {
	return currentPhase == v1.VolumeReady || currentPhase == v1.HotplugVolumeMounted || currentPhase == v1.HotplugVolumeAttachedToNode
}

func (d *VirtualMachineController) setMigrationProgressStatus(vmi *v1.VirtualMachineInstance, domain *api.Domain) *v1.VirtualMachineInstance {

	if domain == nil ||
		domain.Spec.Metadata.KubeVirt.Migration == nil ||
		vmi.Status.MigrationState == nil ||
		!d.isMigrationSource(vmi) {
		return vmi
	}

	migrationMetadata := domain.Spec.Metadata.KubeVirt.Migration
	if migrationMetadata.UID != vmi.Status.MigrationState.MigrationUID {
		return vmi
	}

	if vmi.Status.MigrationState.EndTimestamp == nil && migrationMetadata.EndTimestamp != nil {
		if migrationMetadata.Failed {
			d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Migrated.String(), fmt.Sprintf("VirtualMachineInstance migration uid %s failed. reason:%s", string(migrationMetadata.UID), migrationMetadata.FailureReason))
		}
	}

	if vmi.Status.MigrationState.StartTimestamp == nil {
		vmi.Status.MigrationState.StartTimestamp = migrationMetadata.StartTimestamp
	}
	if vmi.Status.MigrationState.EndTimestamp == nil {
		vmi.Status.MigrationState.EndTimestamp = migrationMetadata.EndTimestamp
	}
	vmi.Status.MigrationState.AbortStatus = v1.MigrationAbortStatus(migrationMetadata.AbortStatus)
	vmi.Status.MigrationState.Completed = migrationMetadata.Completed
	vmi.Status.MigrationState.Failed = migrationMetadata.Failed
	vmi.Status.MigrationState.Mode = migrationMetadata.Mode
	return vmi
}

func (d *VirtualMachineController) migrationSourceUpdateVMIStatus(origVMI *v1.VirtualMachineInstance, domain *api.Domain) error {

	vmi := origVMI.DeepCopy()
	oldStatus := vmi.DeepCopy().Status

	// if a migration happens very quickly, it's possible parts of the in
	// progress status wasn't set. We need to make sure we set this even
	// if the migration has completed
	vmi = d.setMigrationProgressStatus(vmi, domain)

	// handle migrations differently than normal status updates.
	//
	// When a successful migration is detected, we must transfer ownership of the VMI
	// from the source node (this node) to the target node (node the domain was migrated to).
	//
	// Transfer owership by...
	// 1. Marking vmi.Status.MigationState as completed
	// 2. Update the vmi.Status.NodeName to reflect the target node's name
	// 3. Update the VMI's NodeNameLabel annotation to reflect the target node's name
	// 4. Clear the LauncherContainerImageVersion which virt-controller will detect
	//    and accurately based on the version used on the target pod
	//
	// After a migration, the VMI's phase is no longer owned by this node. Only the
	// MigrationState status field is elgible to be mutated.
	migrationHost := ""
	if vmi.Status.MigrationState != nil {
		migrationHost = vmi.Status.MigrationState.TargetNode
	}

	if vmi.Status.MigrationState != nil && vmi.Status.MigrationState.EndTimestamp == nil {
		now := metav1.NewTime(time.Now())
		vmi.Status.MigrationState.EndTimestamp = &now
	}

	targetNodeDetectedDomain, timeLeft := d.hasTargetDetectedDomain(vmi)
	// If we can't detect where the migration went to, then we have no
	// way of transferring ownership. The only option here is to move the
	// vmi to failed.  The cluster vmi controller will then tear down the
	// resulting pods.
	if migrationHost == "" {
		// migrated to unknown host.
		vmi.Status.Phase = v1.Failed
		vmi.Status.MigrationState.Completed = true
		vmi.Status.MigrationState.Failed = true

		d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Migrated.String(), fmt.Sprintf("The VirtualMachineInstance migrated to unknown host."))
	} else if !targetNodeDetectedDomain {
		if timeLeft <= 0 {
			vmi.Status.Phase = v1.Failed
			vmi.Status.MigrationState.Completed = true
			vmi.Status.MigrationState.Failed = true

			d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Migrated.String(), fmt.Sprintf("The VirtualMachineInstance's domain was never observed on the target after the migration completed within the timeout period."))
		} else {
			log.Log.Object(vmi).Info("Waiting on the target node to observe the migrated domain before performing the handoff")
		}
	} else if vmi.Status.MigrationState != nil && vmi.Status.MigrationState.TargetNodeDomainDetected {
		// this is the migration ACK.
		// At this point we know that the migration has completed and that
		// the target node has seen the domain event.
		vmi.Labels[v1.NodeNameLabel] = migrationHost
		delete(vmi.Labels, v1.OutdatedLauncherImageLabel)
		vmi.Status.LauncherContainerImageVersion = ""
		vmi.Status.NodeName = migrationHost
		// clean the evacuation node name since have already migrated to a new node
		vmi.Status.EvacuationNodeName = ""
		vmi.Status.MigrationState.Completed = true
		d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.Migrated.String(), fmt.Sprintf("The VirtualMachineInstance migrated to node %s.", migrationHost))
		log.Log.Object(vmi).Infof("migration completed to node %s", migrationHost)
	}

	if !reflect.DeepEqual(oldStatus, vmi.Status) {
		_, err := d.clientset.VirtualMachineInstance(vmi.ObjectMeta.Namespace).Update(vmi)
		if err != nil {
			return err
		}
	}
	return nil
}

func (d *VirtualMachineController) migrationTargetUpdateVMIStatus(vmi *v1.VirtualMachineInstance, domainExists bool) error {

	vmiCopy := vmi.DeepCopy()

	if migrations.MigrationFailed(vmi) {
		// nothing left to report on the target node if the migration failed
		return nil
	}

	// Handle post migration
	if domainExists && vmi.Status.MigrationState != nil && !vmi.Status.MigrationState.TargetNodeDomainDetected {
		// record that we've see the domain populated on the target's node
		log.Log.Object(vmi).Info("The target node received the migrated domain")
		vmiCopy.Status.MigrationState.TargetNodeDomainDetected = true
		d.finalizeMigration(vmi)
	}

	if !migrations.IsMigrating(vmi) {
		destSrcPortsMap := d.migrationProxy.GetTargetListenerPorts(string(vmi.UID))
		if len(destSrcPortsMap) == 0 {
			msg := "target migration listener is not up for this vmi"
			log.Log.Object(vmi).Error(msg)
			return fmt.Errorf(msg)
		}

		hostAddress := ""

		// advertise the listener address to the source node
		if vmi.Status.MigrationState != nil {
			hostAddress = vmi.Status.MigrationState.TargetNodeAddress
		}
		if hostAddress != d.ipAddress {
			portsList := make([]string, 0, len(destSrcPortsMap))

			for k := range destSrcPortsMap {
				portsList = append(portsList, k)
			}
			portsStrList := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(portsList)), ","), "[]")
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.PreparingTarget.String(), fmt.Sprintf("Migration Target is listening at %s, on ports: %s", d.ipAddress, portsStrList))
			vmiCopy.Status.MigrationState.TargetNodeAddress = d.ipAddress
			vmiCopy.Status.MigrationState.TargetDirectMigrationNodePorts = destSrcPortsMap
		}
	}

	// update the VMI if necessary
	if !reflect.DeepEqual(vmi.Status, vmiCopy.Status) {
		_, err := d.clientset.VirtualMachineInstance(vmi.ObjectMeta.Namespace).Update(vmiCopy)
		if err != nil {
			return err
		}
	}

	return nil
}

func (d *VirtualMachineController) generateEventsForVolumeStatusChange(vmi *v1.VirtualMachineInstance, newStatusMap map[string]v1.VolumeStatus) {
	newStatusMapCopy := make(map[string]v1.VolumeStatus)
	for k, v := range newStatusMap {
		newStatusMapCopy[k] = v
	}
	for _, oldStatus := range vmi.Status.VolumeStatus {
		newStatus, ok := newStatusMap[oldStatus.Name]
		if !ok {
			// status got removed
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, VolumeUnplugged, fmt.Sprintf("Volume %s has been unplugged", oldStatus.Name))
			continue
		}
		if newStatus.Phase != oldStatus.Phase {
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, newStatus.Reason, newStatus.Message)
		}
		delete(newStatusMapCopy, newStatus.Name)
	}
	// Send events for any new statuses.
	for _, v := range newStatusMapCopy {
		d.recorder.Event(vmi, k8sv1.EventTypeNormal, v.Reason, v.Message)
	}
}

func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMachineInstance, volumeStatus v1.VolumeStatus, specVolumeMap map[string]v1.Volume) (v1.VolumeStatus, bool) {
	needsRefresh := false
	if volumeStatus.Target == "" {
		needsRefresh = true
		if mounted, _ := d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID); mounted {
			if _, ok := specVolumeMap[volumeStatus.Name]; ok && canUpdateToMounted(volumeStatus.Phase) {
				log.DefaultLogger().Infof("Marking volume %s as mounted in pod, it can now be attached", volumeStatus.Name)
				// mounted, and still in spec, and in phase we can change, update status to mounted.
				volumeStatus.Phase = v1.HotplugVolumeMounted
				volumeStatus.Message = fmt.Sprintf("Volume %s has been mounted in virt-launcher pod", volumeStatus.Name)
				volumeStatus.Reason = VolumeMountedToPodReason
			}
		} else {
			// Not mounted, check if the volume is in the spec, if not update status
			if _, ok := specVolumeMap[volumeStatus.Name]; !ok && canUpdateToUnmounted(volumeStatus.Phase) {
				log.DefaultLogger().Infof("Marking volume %s as unmounted from pod, it can now be detached", volumeStatus.Name)
				// Not mounted.
				volumeStatus.Phase = v1.HotplugVolumeUnMounted
				volumeStatus.Message = fmt.Sprintf("Volume %s has been unmounted from virt-launcher pod", volumeStatus.Name)
				volumeStatus.Reason = VolumeUnMountedFromPodReason
			}
		}
	} else {
		// Successfully attached to VM.
		volumeStatus.Phase = v1.VolumeReady
		volumeStatus.Message = fmt.Sprintf("Successfully attach hotplugged volume %s to VM", volumeStatus.Name)
		volumeStatus.Reason = VolumeReadyReason
	}
	return volumeStatus, needsRefresh
}

func (d *VirtualMachineController) updateVolumeStatusesFromDomain(vmi *v1.VirtualMachineInstance, domain *api.Domain) bool {
	hasHotplug := false
	if len(vmi.Status.VolumeStatus) > 0 {
		diskDeviceMap := make(map[string]string)
		for _, disk := range domain.Spec.Devices.Disks {
			diskDeviceMap[disk.Alias.GetName()] = disk.Target.Device
		}
		specVolumeMap := make(map[string]v1.Volume)
		for _, volume := range vmi.Spec.Volumes {
			specVolumeMap[volume.Name] = volume
		}
		newStatusMap := make(map[string]v1.VolumeStatus)
		newStatuses := make([]v1.VolumeStatus, 0)
		needsRefresh := false
		for _, volumeStatus := range vmi.Status.VolumeStatus {
			if _, ok := diskDeviceMap[volumeStatus.Name]; ok {
				volumeStatus.Target = diskDeviceMap[volumeStatus.Name]
			}
			if volumeStatus.HotplugVolume != nil {
				hasHotplug = true
				volumeStatus, needsRefresh = d.updateHotplugVolumeStatus(vmi, volumeStatus, specVolumeMap)
			}
			newStatuses = append(newStatuses, volumeStatus)
			newStatusMap[volumeStatus.Name] = volumeStatus
		}
		sort.SliceStable(newStatuses, func(i, j int) bool {
			return strings.Compare(newStatuses[i].Name, newStatuses[j].Name) == -1
		})
		if needsRefresh {
			d.Queue.AddAfter(controller.VirtualMachineKey(vmi), time.Second)
		}
		d.generateEventsForVolumeStatusChange(vmi, newStatusMap)
		vmi.Status.VolumeStatus = newStatuses
	}
	return hasHotplug
}

func (d *VirtualMachineController) updateVMIStatus(origVMI *v1.VirtualMachineInstance, domain *api.Domain, syncError error) (err error) {
	condManager := controller.NewVirtualMachineInstanceConditionManager()
	hasHotplug := false

	// Don't update the VirtualMachineInstance if it is already in a final state
	if origVMI.IsFinal() {
		return nil
	} else if origVMI.Status.NodeName != "" && origVMI.Status.NodeName != d.host {
		// Only update the VMI's phase if this node owns the VMI.
		// not owned by this host, likely the result of a migration
		return nil
	} else if domainMigrated(domain) {
		return d.migrationSourceUpdateVMIStatus(origVMI, domain)
	}

	vmi := origVMI.DeepCopy()
	oldStatus := vmi.DeepCopy().Status

	vmi = d.setMigrationProgressStatus(vmi, domain)

	if domain != nil {
		if vmi.Status.GuestOSInfo.Name != domain.Status.OSInfo.Name {
			vmi.Status.GuestOSInfo.Name = domain.Status.OSInfo.Name
			vmi.Status.GuestOSInfo.Version = domain.Status.OSInfo.VersionId
			vmi.Status.GuestOSInfo.KernelRelease = domain.Status.OSInfo.KernelRelease
			vmi.Status.GuestOSInfo.PrettyName = domain.Status.OSInfo.PrettyName
			vmi.Status.GuestOSInfo.VersionID = domain.Status.OSInfo.VersionId
			vmi.Status.GuestOSInfo.KernelVersion = domain.Status.OSInfo.KernelVersion
			vmi.Status.GuestOSInfo.ID = domain.Status.OSInfo.Id
		}
		// This is needed to be backwards compatible with vmi's which have status interfaces
		// with the name not being set
		if len(domain.Spec.Devices.Interfaces) == 0 && len(vmi.Status.Interfaces) == 1 && vmi.Status.Interfaces[0].Name == "" {
			for _, network := range vmi.Spec.Networks {
				if network.NetworkSource.Pod != nil {
					vmi.Status.Interfaces[0].Name = network.Name
				}
			}
		}

		hasHotplug = d.updateVolumeStatusesFromDomain(vmi, domain)
		if len(vmi.Status.Interfaces) == 0 {
			// Set Pod Interface
			interfaces := make([]v1.VirtualMachineInstanceNetworkInterface, 0)
			for _, network := range vmi.Spec.Networks {
				if network.NetworkSource.Pod != nil {
					podIface, err := d.getPodInterfacefromFileCache(vmi, network.Name)
					if err != nil {
						return err
					}
					ifc := v1.VirtualMachineInstanceNetworkInterface{
						Name: network.Name,
						IP:   podIface.PodIP,
						IPs:  podIface.PodIPs,
					}
					interfaces = append(interfaces, ifc)
				}
			}
			vmi.Status.Interfaces = interfaces
		}

		if len(domain.Spec.Devices.Interfaces) > 0 || len(domain.Status.Interfaces) > 0 {
			// This calculates the vmi.Status.Interfaces based on the following data sets:
			// - vmi.Status.Interfaces - previously calculated interfaces, this can contain data (pod IP)
			//   set in the previous loops (when there are no interfaces), which can not be deleted,
			//   unless overridden by Qemu agent
			// - domain.Spec - interfaces form the Spec
			// - domain.Status.Interfaces - interfaces reported by guest agent (empty if Qemu agent not running)
			newInterfaces := []v1.VirtualMachineInstanceNetworkInterface{}

			existingInterfaceStatusByName := map[string]v1.VirtualMachineInstanceNetworkInterface{}
			for _, existingInterfaceStatus := range vmi.Status.Interfaces {
				if existingInterfaceStatus.Name != "" {
					existingInterfaceStatusByName[existingInterfaceStatus.Name] = existingInterfaceStatus
				}
			}

			domainInterfaceStatusByMac := map[string]api.InterfaceStatus{}
			for _, domainInterfaceStatus := range domain.Status.Interfaces {
				domainInterfaceStatusByMac[domainInterfaceStatus.Mac] = domainInterfaceStatus
			}

			existingInterfacesSpecByName := map[string]v1.Interface{}
			for _, existingInterfaceSpec := range vmi.Spec.Domain.Devices.Interfaces {
				existingInterfacesSpecByName[existingInterfaceSpec.Name] = existingInterfaceSpec
			}
			existingNetworksByName := map[string]v1.Network{}
			for _, existingNetwork := range vmi.Spec.Networks {
				existingNetworksByName[existingNetwork.Name] = existingNetwork
			}

			// Iterate through all domain.Spec interfaces
			for _, domainInterface := range domain.Spec.Devices.Interfaces {
				interfaceMAC := domainInterface.MAC.MAC
				var newInterface v1.VirtualMachineInstanceNetworkInterface
				var isForwardingBindingInterface = false

				if existingInterfacesSpecByName[domainInterface.Alias.GetName()].Masquerade != nil || existingInterfacesSpecByName[domainInterface.Alias.GetName()].Slirp != nil {
					isForwardingBindingInterface = true
				}

				if existingInterface, exists := existingInterfaceStatusByName[domainInterface.Alias.GetName()]; exists {
					// Reuse previously calculated interface from vmi.Status.Interfaces, updating the MAC from domain.Spec
					// Only interfaces defined in domain.Spec are handled here
					newInterface = existingInterface
					newInterface.MAC = interfaceMAC

					// If it is a Combination of Masquerade+Pod network, check IP from file cache
					if existingInterfacesSpecByName[domainInterface.Alias.GetName()].Masquerade != nil && existingNetworksByName[domainInterface.Alias.GetName()].NetworkSource.Pod != nil {
						iface, err := d.getPodInterfacefromFileCache(vmi, domainInterface.Alias.GetName())
						if err != nil {
							return err
						}

						if !reflect.DeepEqual(iface.PodIPs, existingInterfaceStatusByName[domainInterface.Alias.GetName()].IPs) {
							newInterface.Name = domainInterface.Alias.GetName()
							newInterface.IP = iface.PodIP
							newInterface.IPs = iface.PodIPs
						}
					}
				} else {
					// If not present in vmi.Status.Interfaces, create a new one based on domain.Spec
					newInterface = v1.VirtualMachineInstanceNetworkInterface{
						MAC:  interfaceMAC,
						Name: domainInterface.Alias.GetName(),
					}
				}

				// Update IP info based on information from domain.Status.Interfaces (Qemu guest)
				// Remove the interface from domainInterfaceStatusByMac to mark it as handled
				if interfaceStatus, exists := domainInterfaceStatusByMac[interfaceMAC]; exists {
					newInterface.InterfaceName = interfaceStatus.InterfaceName
					// Do not update if interface has Masquerede binding
					// virt-controller should update VMI status interface with Pod IP instead
					if !isForwardingBindingInterface {
						newInterface.IP = interfaceStatus.Ip
						newInterface.IPs = interfaceStatus.IPs
					}
					delete(domainInterfaceStatusByMac, interfaceMAC)
				}
				newInterfaces = append(newInterfaces, newInterface)
			}

			// If any of domain.Status.Interfaces were not handled above, it means that the vm contains additional
			// interfaces not defined in domain.Spec.Devices.Interfaces (most likely added by user on VM or a SRIOV interface)
			// Add them to vmi.Status.Interfaces
			setMissingSRIOVInterfacesNames(existingInterfacesSpecByName, domainInterfaceStatusByMac)
			for interfaceMAC, domainInterfaceStatus := range domainInterfaceStatusByMac {
				newInterface := v1.VirtualMachineInstanceNetworkInterface{
					Name:          domainInterfaceStatus.Name,
					MAC:           interfaceMAC,
					IP:            domainInterfaceStatus.Ip,
					IPs:           domainInterfaceStatus.IPs,
					InterfaceName: domainInterfaceStatus.InterfaceName,
				}
				newInterfaces = append(newInterfaces, newInterface)
			}
			vmi.Status.Interfaces = newInterfaces
		}
	}

	// Update AccessCredential conditions
	if domain != nil && domain.Spec.Metadata.KubeVirt.AccessCredential != nil {

		message := domain.Spec.Metadata.KubeVirt.AccessCredential.Message
		status := k8sv1.ConditionFalse
		if domain.Spec.Metadata.KubeVirt.AccessCredential.Succeeded {
			status = k8sv1.ConditionTrue
		}

		add := false
		condition := condManager.GetCondition(vmi, v1.VirtualMachineInstanceAccessCredentialsSynchronized)
		if condition == nil {
			add = true
		} else if condition.Status != status || condition.Message != message {
			// if not as expected, remove, then add.
			condManager.RemoveCondition(vmi, v1.VirtualMachineInstanceAccessCredentialsSynchronized)
			add = true
		}
		if add {
			newCondition := v1.VirtualMachineInstanceCondition{
				Type:               v1.VirtualMachineInstanceAccessCredentialsSynchronized,
				LastTransitionTime: metav1.Now(),
				Status:             status,
				Message:            message,
			}
			vmi.Status.Conditions = append(vmi.Status.Conditions, newCondition)
			if status == k8sv1.ConditionTrue {
				d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.AccessCredentialsSyncSuccess.String(), message)
			} else {
				d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.AccessCredentialsSyncFailed.String(), message)
			}
		}
	}

	// Calculate the new VirtualMachineInstance state based on what libvirt reported
	err = d.setVmPhaseForStatusReason(domain, vmi)
	if err != nil {
		return err
	}

	// Cacluate whether the VM is migratable
	liveMigrationCondition, isBlockMigration := d.calculateLiveMigrationCondition(vmi, hasHotplug)
	if !condManager.HasCondition(vmi, v1.VirtualMachineInstanceIsMigratable) {
		vmi.Status.Conditions = append(vmi.Status.Conditions, *liveMigrationCondition)
		// Set VMI Migration Method
		if isBlockMigration {
			vmi.Status.MigrationMethod = v1.BlockMigration
		} else {
			vmi.Status.MigrationMethod = v1.LiveMigration
		}
	} else {
		cond := condManager.GetCondition(vmi, v1.VirtualMachineInstanceIsMigratable)
		if !reflect.DeepEqual(cond, liveMigrationCondition) {
			condManager.RemoveCondition(vmi, v1.VirtualMachineInstanceIsMigratable)
			vmi.Status.Conditions = append(vmi.Status.Conditions, *liveMigrationCondition)
		}
	}
	if vmi.IsEvictable() && liveMigrationCondition.Status == k8sv1.ConditionFalse {
		d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Migrated.String(), "EvictionStrategy is set but vmi is not migratable")
	}
	// Update the condition when GA is connected
	channelConnected := false
	if domain != nil {
		for _, channel := range domain.Spec.Devices.Channels {
			if channel.Target != nil {
				log.Log.V(4).Infof("Channel: %s, %s", channel.Target.Name, channel.Target.State)
				if channel.Target.Name == "org.qemu.guest_agent.0" {
					if channel.Target.State == "connected" {
						channelConnected = true
					}
				}

			}
		}
	}

	switch {
	case channelConnected && !condManager.HasCondition(vmi, v1.VirtualMachineInstanceAgentConnected):
		agentCondition := v1.VirtualMachineInstanceCondition{
			Type:          v1.VirtualMachineInstanceAgentConnected,
			LastProbeTime: metav1.Now(),
			Status:        k8sv1.ConditionTrue,
		}
		vmi.Status.Conditions = append(vmi.Status.Conditions, agentCondition)
	case !channelConnected:
		condManager.RemoveCondition(vmi, v1.VirtualMachineInstanceAgentConnected)
	}

	if condManager.HasCondition(vmi, v1.VirtualMachineInstanceAgentConnected) {
		client, err := d.getLauncherClient(vmi)
		if err != nil {
			return err
		}

		guestInfo, err := client.GetGuestInfo()
		if err != nil {
			return err
		}

		var supported = false

		// For current versions, virt-launcher's supported commands will always contain data.
		// For backwards compatibility: during upgrade from a previous version of KubeVirt,
		// virt-launcher might not provide any supported commands. If the list of supported
		// commands is empty, fall back to previous behavior.
		if len(guestInfo.SupportedCommands) > 0 {
			supported = isGuestAgentSupported(vmi, guestInfo.SupportedCommands)
		} else {
			for _, version := range d.clusterConfig.GetSupportedAgentVersions() {
				supported = supported || regexp.MustCompile(version).MatchString(guestInfo.GAVersion)
			}
		}

		if !supported {
			if !condManager.HasCondition(vmi, v1.VirtualMachineInstanceUnsupportedAgent) {
				agentCondition := v1.VirtualMachineInstanceCondition{
					Type:          v1.VirtualMachineInstanceUnsupportedAgent,
					LastProbeTime: metav1.Now(),
					Status:        k8sv1.ConditionTrue,
				}
				vmi.Status.Conditions = append(vmi.Status.Conditions, agentCondition)
			}
		} else {
			condManager.RemoveCondition(vmi, v1.VirtualMachineInstanceUnsupportedAgent)
		}

	}

	//
	// Update paused condition in case VMI was paused / unpaused
	if domain != nil && domain.Status.Status == api.Paused {
		if !condManager.HasCondition(vmi, v1.VirtualMachineInstancePaused) {
			calculatePausedCondition(vmi, domain.Status.Reason)
		}
	} else if condManager.HasCondition(vmi, v1.VirtualMachineInstancePaused) {
		log.Log.Object(vmi).V(3).Info("Removing paused condition")
		condManager.RemoveCondition(vmi, v1.VirtualMachineInstancePaused)
	}

	if _, ok := syncError.(*virtLauncherCriticalNetworkError); ok {
		log.Log.Errorf("virt-launcher crashed due to a network error. Updating VMI %s status to Failed", vmi.Name)
		vmi.Status.Phase = v1.Failed
	}
	if _, ok := syncError.(*virtLauncherCriticalSecurebootError); ok {
		log.Log.Errorf("virt-launcher does not support the Secure Boot setting. Updating VMI %s status to Failed", vmi.Name)
		vmi.Status.Phase = v1.Failed
	}
	condManager.CheckFailure(vmi, syncError, "Synchronizing with the Domain failed.")

	if !reflect.DeepEqual(oldStatus, vmi.Status) {
		_, err = d.clientset.VirtualMachineInstance(vmi.ObjectMeta.Namespace).Update(vmi)
		if err != nil {
			return err
		}
	}

	if oldStatus.Phase != vmi.Status.Phase {
		switch vmi.Status.Phase {
		case v1.Running:
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.Started.String(), VMIStarted)
		case v1.Succeeded:
			d.recorder.Event(vmi, k8sv1.EventTypeNormal, v1.Stopped.String(), VMIShutdown)
		case v1.Failed:
			d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Stopped.String(), VMICrashed)
		}
	}

	return nil
}

func _guestAgentCommandSubsetSupported(requiredCommands []string, commands []v1.GuestAgentCommandInfo) bool {
	var found bool
	for _, cmd := range requiredCommands {
		found = false
		for _, foundCmd := range commands {
			if cmd == foundCmd.Name {
				if foundCmd.Enabled {
					found = true
				}
				break
			}
		}
		if found == false {
			return false
		}
	}
	return true

}

func isGuestAgentSupported(vmi *v1.VirtualMachineInstance, commands []v1.GuestAgentCommandInfo) bool {
	log.Log.V(3).Object(vmi).Infof("checking guest agent: %v", commands)
	if !_guestAgentCommandSubsetSupported(RequiredGuestAgentCommands, commands) {
		log.Log.V(3).Object(vmi).Info("This guest agent doesn't support required basic commands")
		return false
	}

	checkSSH := false
	checkPasswd := false

	if vmi != nil && vmi.Spec.AccessCredentials != nil {
		for _, accessCredential := range vmi.Spec.AccessCredentials {
			if accessCredential.SSHPublicKey != nil && accessCredential.SSHPublicKey.PropagationMethod.QemuGuestAgent != nil {
				// defer checking the command list so we only do that once
				checkSSH = true
			}
			if accessCredential.UserPassword != nil && accessCredential.UserPassword.PropagationMethod.QemuGuestAgent != nil {
				// defer checking the command list so we only do that once
				checkPasswd = true
			}

		}
	}

	if checkSSH && !_guestAgentCommandSubsetSupported(SSHRelatedGuestAgentCommands, commands) {
		log.Log.V(3).Object(vmi).Info("This guest agent doesn't support required public key commands")
		return false
	}

	if checkPasswd && !_guestAgentCommandSubsetSupported(PasswordRelatedGuestAgentCommands, commands) {
		log.Log.V(3).Object(vmi).Info("This guest agent doesn't support required password commands")
		return false
	}

	log.Log.V(3).Object(vmi).Info("This guest agent is supported")
	return true
}

func calculatePausedCondition(vmi *v1.VirtualMachineInstance, reason api.StateChangeReason) {
	if reason == api.ReasonPausedUser {
		log.Log.Object(vmi).V(3).Info("Adding paused condition")
		now := metav1.NewTime(time.Now())
		vmi.Status.Conditions = append(vmi.Status.Conditions, v1.VirtualMachineInstanceCondition{
			Type:               v1.VirtualMachineInstancePaused,
			Status:             k8sv1.ConditionTrue,
			LastProbeTime:      now,
			LastTransitionTime: now,
			Reason:             "PausedByUser",
			Message:            "VMI was paused by user",
		})
	}
}
{'fraction_non_alphanumeric': 0.09705408692032122, 'fraction_numerical': 0.005598220839404464, 'mean_word_length': 4.0906251742791815, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4588775', 'n_tokens_mistral': 30967, 'n_tokens_neox': 28164, 'n_words': 6849}