<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017, 2018 Red Hat, Inc.
 *
 */

package virtwrap

//go:generate mockgen -source $GOFILE -package=$GOPACKAGE -destination=generated_mock_$GOFILE

/*
 ATTENTION: Rerun code generators when interface signatures are modified.
*/

import (
	"context"
	"encoding/xml"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/generic"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/gpu"

	"kubevirt.io/kubevirt/pkg/downwardmetrics"
	"kubevirt.io/kubevirt/pkg/network/cache"
	"kubevirt.io/kubevirt/pkg/util/hardware"
	cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/agent"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/converter"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/efi"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"libvirt.org/go/libvirt"

	v1 "kubevirt.io/api/core/v1"
	"kubevirt.io/client-go/log"
	cloudinit "kubevirt.io/kubevirt/pkg/cloud-init"
	"kubevirt.io/kubevirt/pkg/config"
	containerdisk "kubevirt.io/kubevirt/pkg/container-disk"
	"kubevirt.io/kubevirt/pkg/controller"
	"kubevirt.io/kubevirt/pkg/emptydisk"
	ephemeraldisk "kubevirt.io/kubevirt/pkg/ephemeral-disk"
	cmdv1 "kubevirt.io/kubevirt/pkg/handler-launcher-com/cmd/v1"
	"kubevirt.io/kubevirt/pkg/hooks"
	"kubevirt.io/kubevirt/pkg/ignition"
	netsetup "kubevirt.io/kubevirt/pkg/network/setup"
	netsriov "kubevirt.io/kubevirt/pkg/network/sriov"
	kutil "kubevirt.io/kubevirt/pkg/util"
	accesscredentials "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/access-credentials"
	agentpoller "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/agent-poller"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/cli"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/sriov"
	domainerrors "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/errors"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/util"
)

const (
	failedSyncGuestTime  = "failed to sync guest time"
	failedGetDomain      = "Getting the domain failed."
	failedGetDomainState = "Getting the domain state failed."
)

const (
	PCI_RESOURCE_PREFIX  = "PCI_RESOURCE"
	MDEV_RESOURCE_PREFIX = "MDEV_PCI_RESOURCE"

	baseEphemeralDiskDir = "/var/run/kubevirt-ephemeral-disks/"
)

const maxConcurrentHotplugHostDevices = 1

var (
	getEphemeralDiskBaseDir = func() string {
		return baseEphemeralDiskDir
	}
)

type contextStore struct {
	ctx    context.Context
	cancel context.CancelFunc
}

type DomainManager interface {
	SyncVMI(*v1.VirtualMachineInstance, bool, *cmdv1.VirtualMachineOptions) (*api.DomainSpec, error)
	PauseVMI(*v1.VirtualMachineInstance) error
	UnpauseVMI(*v1.VirtualMachineInstance) error
	FreezeVMI(*v1.VirtualMachineInstance, int32) error
	UnfreezeVMI(*v1.VirtualMachineInstance) error
	SoftRebootVMI(*v1.VirtualMachineInstance) error
	KillVMI(*v1.VirtualMachineInstance) error
	DeleteVMI(*v1.VirtualMachineInstance) error
	SignalShutdownVMI(*v1.VirtualMachineInstance) error
	MarkGracefulShutdownVMI(*v1.VirtualMachineInstance) error
	ListAllDomains() ([]*api.Domain, error)
	MigrateVMI(*v1.VirtualMachineInstance, *cmdclient.MigrationOptions) error
	PrepareMigrationTarget(*v1.VirtualMachineInstance, bool, *cmdv1.VirtualMachineOptions) error
	GetDomainStats() ([]*stats.DomainStats, error)
	CancelVMIMigration(*v1.VirtualMachineInstance) error
	GetGuestInfo() (v1.VirtualMachineInstanceGuestAgentInfo, error)
	GetUsers() ([]v1.VirtualMachineInstanceGuestOSUser, error)
	GetFilesystems() ([]v1.VirtualMachineInstanceFileSystem, error)
	FinalizeVirtualMachineMigration(*v1.VirtualMachineInstance) error
	HotplugHostDevices(vmi *v1.VirtualMachineInstance) error
	InterfacesStatus() []api.InterfaceStatus
	GetGuestOSInfo() *api.GuestOSInfo
	Exec(string, string, []string, int32) (string, error)
	GuestPing(string) error
}

type LibvirtDomainManager struct {
	virConn cli.Connection

	// Anytime a get and a set is done on the domain, this lock must be held.
	domainModifyLock sync.Mutex
	// mutex to control access to the guest time context
	setGuestTimeLock sync.Mutex

	credManager *accesscredentials.AccessCredentialManager

	hotplugHostDevicesInProgress chan struct{}

	virtShareDir             string
	paused                   pausedVMIs
	agentData                *agentpoller.AsyncAgentStore
	cloudInitDataStore       *cloudinit.CloudInitData
	setGuestTimeContextPtr   *contextStore
	efiEnvironment           *efi.EFIEnvironment
	ovmfPath                 string
	ephemeralDiskCreator     ephemeraldisk.EphemeralDiskCreatorInterface
	directIOChecker          converter.DirectIOChecker
	disksInfo                map[string]*cmdv1.DiskInfo
	cancelSafetyUnfreezeChan chan struct{}
}

type pausedVMIs struct {
	paused map[types.UID]bool
}

func (s pausedVMIs) add(uid types.UID) {
	// implicitly locked by domainModifyLock
	if _, ok := s.paused[uid]; !ok {
		s.paused[uid] = true
	}
}

func (s pausedVMIs) remove(uid types.UID) {
	// implicitly locked by domainModifyLock
	if _, ok := s.paused[uid]; ok {
		delete(s.paused, uid)
	}
}

func (s pausedVMIs) contains(uid types.UID) bool {
	_, ok := s.paused[uid]
	return ok
}

func NewLibvirtDomainManager(connection cli.Connection, virtShareDir string, agentStore *agentpoller.AsyncAgentStore, ovmfPath string, ephemeralDiskCreator ephemeraldisk.EphemeralDiskCreatorInterface) (DomainManager, error) {
	directIOChecker := converter.NewDirectIOChecker()
	return newLibvirtDomainManager(connection, virtShareDir, agentStore, ovmfPath, ephemeralDiskCreator, directIOChecker)
}

func newLibvirtDomainManager(connection cli.Connection, virtShareDir string, agentStore *agentpoller.AsyncAgentStore, ovmfPath string, ephemeralDiskCreator ephemeraldisk.EphemeralDiskCreatorInterface, directIOChecker converter.DirectIOChecker) (DomainManager, error) {
	manager := LibvirtDomainManager{
		virConn:      connection,
		virtShareDir: virtShareDir,
		paused: pausedVMIs{
			paused: make(map[types.UID]bool, 0),
		},
		agentData:                agentStore,
		efiEnvironment:           efi.DetectEFIEnvironment(runtime.GOARCH, ovmfPath),
		ephemeralDiskCreator:     ephemeralDiskCreator,
		directIOChecker:          directIOChecker,
		disksInfo:                map[string]*cmdv1.DiskInfo{},
		cancelSafetyUnfreezeChan: make(chan struct{}),
	}

	manager.hotplugHostDevicesInProgress = make(chan struct{}, maxConcurrentHotplugHostDevices)
	manager.credManager = accesscredentials.NewManager(connection, &manager.domainModifyLock)

	return &manager, nil
}

func getDomainSpec(dom cli.VirDomain) (*api.DomainSpec, error) {
	var newSpec api.DomainSpec
	xmlstr, err := dom.GetXMLDesc(0)
	if err != nil {
		return &newSpec, err
	}
	err = xml.Unmarshal([]byte(xmlstr), &newSpec)
	if err != nil {
		return &newSpec, err
	}

	return &newSpec, nil
}

func getAllDomainDevices(dom cli.VirDomain) (api.Devices, error) {
	domSpec, err := getDomainSpec(dom)
	if err != nil {
		return domSpec.Devices, err
	}
	return domSpec.Devices, nil
}

func getAllDomainDisks(dom cli.VirDomain) ([]api.Disk, error) {
	devices, err := getAllDomainDevices(dom)
	if err != nil {
		return nil, err
	}

	return devices.Disks, nil
}

func (l *LibvirtDomainManager) setGuestTime(vmi *v1.VirtualMachineInstance) error {
	// Try to set VM time to the current value.  This is typically useful
	// when clock wasn't running on the VM for some time (e.g. during
	// suspension or migration), especially if the time delay exceeds NTP
	// tolerance.
	// It is not guaranteed that the time is actually set (it depends on guest
	// environment, especially QEMU agent presence) or that the set time is
	// very precise (NTP in the guest should take care of it if needed).

	domName := api.VMINamespaceKeyFunc(vmi)
	dom, err := l.virConn.LookupDomainByName(domName)
	if err != nil {
		log.Log.Object(vmi).Reason(err).Error(failedSyncGuestTime)
		return err
	}

	go func() {
		defer dom.Free()

		ctx := l.getGuestTimeContext()
		timeout := time.After(60 * time.Second)
		ticker := time.NewTicker(time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-timeout:
				log.Log.Object(vmi).Error(failedSyncGuestTime)
				return
			case <-ctx.Done():
				return
			case <-ticker.C:
				currTime := time.Now()
				secs := currTime.Unix()
				nsecs := uint(currTime.Nanosecond())
				err := dom.SetTime(secs, nsecs, 0)
				if err != nil {
					libvirtError, ok := err.(libvirt.Error)
					if !ok {
						log.Log.Object(vmi).Reason(err).Warning(failedSyncGuestTime)
						return
					}

					switch libvirtError.Code {
					case libvirt.ERR_AGENT_UNRESPONSIVE:
						log.Log.Object(vmi).Reason(err).Warning("failed to set time: QEMU agent unresponsive")
					case libvirt.ERR_OPERATION_UNSUPPORTED:
						// no need to retry as this opertaion is not supported
						log.Log.Object(vmi).Reason(err).Warning("failed to set time: not supported")
						return
					case libvirt.ERR_ARGUMENT_UNSUPPORTED:
						// no need to retry as the agent is not configured
						log.Log.Object(vmi).Reason(err).Warning("failed to set time: agent not configured")
						return
					default:
						log.Log.Object(vmi).Reason(err).Warning(failedSyncGuestTime)
					}
				} else {
					log.Log.Object(vmi).Info("guest VM time sync finished successfully")
					return
				}
			}
		}
	}()

	return nil
}

func (l *LibvirtDomainManager) getGuestTimeContext() context.Context {
	l.setGuestTimeLock.Lock()
	defer l.setGuestTimeLock.Unlock()

	// cancel the already running setGuestTime go-routine if such exist
	if l.setGuestTimeContextPtr != nil {
		l.setGuestTimeContextPtr.cancel()
	}
	// create a new context and store it
	ctx, cancel := context.WithCancel(context.Background())
	l.setGuestTimeContextPtr = &contextStore{ctx: ctx, cancel: cancel}
	return ctx
}

// PrepareMigrationTarget the target pod environment before the migration is initiated
func (l *LibvirtDomainManager) PrepareMigrationTarget(
	vmi *v1.VirtualMachineInstance,
	allowEmulation bool,
	options *cmdv1.VirtualMachineOptions,
) error {
	return l.prepareMigrationTarget(vmi, allowEmulation, options)
}

// FinalizeVirtualMachineMigration finalized the migration after the migration has completed and vmi is running on target pod.
func (l *LibvirtDomainManager) FinalizeVirtualMachineMigration(vmi *v1.VirtualMachineInstance) error {
	return l.finalizeMigrationTarget(vmi)
}

// HotplugHostDevices attach host-devices to running domain, currently only SRIOV host-devices are supported.
// This operation runs in the background, only one hotplug operation can occur at a time.
func (l *LibvirtDomainManager) HotplugHostDevices(vmi *v1.VirtualMachineInstance) error {
	select {
	case l.hotplugHostDevicesInProgress <- struct{}{}:
	default:
		return fmt.Errorf("hot-plug host-devices is in progress")
	}

	go func() {
		defer func() { <-l.hotplugHostDevicesInProgress }()

		if err := l.hotPlugHostDevices(vmi); err != nil {
			log.Log.Object(vmi).Error(err.Error())
		}
	}()
	return nil
}

func (l *LibvirtDomainManager) hotPlugHostDevices(vmi *v1.VirtualMachineInstance) error {
	l.domainModifyLock.Lock()
	defer l.domainModifyLock.Unlock()

	const errMsgPrefix = "failed to hot-plug host-devices"

	domainName := api.VMINamespaceKeyFunc(vmi)
	domain, err := l.virConn.LookupDomainByName(domainName)
	if err != nil {
		return fmt.Errorf("%s: %v", errMsgPrefix, err)
	}
	defer domain.Free()

	domainSpec, err := util.GetDomainSpecWithFlags(domain, 0)
	if err != nil {
		return fmt.Errorf("%s: %v", errMsgPrefix, err)
	}

	sriovHostDevices, err := sriov.GetHostDevicesToAttach(vmi, domainSpec)
	if err != nil {
		return fmt.Errorf("%s: %v", errMsgPrefix, err)
	}

	if err := hostdevice.AttachHostDevices(domain, sriovHostDevices); err != nil {
		return fmt.Errorf("%s: %v", errMsgPrefix, hostdevice.AttachHostDevices(domain, sriovHostDevices))
	}

	return nil
}

func (l *LibvirtDomainManager) Exec(domainName, command string, args []string, timeoutSeconds int32) (string, error) {
	return agent.GuestExec(l.virConn, domainName, command, args, timeoutSeconds)
}

func (l *LibvirtDomainManager) GuestPing(domainName string) error {
	pingCmd := `{"execute":"guest-ping"}`
	_, err := l.virConn.QemuAgentCommand(pingCmd, domainName)
	return err
}

func getVMIEphemeralDisksTotalSize() *resource.Quantity {
	baseDir := getEphemeralDiskBaseDir()
	totalSize := int64(0)
	err := filepath.Walk(baseDir, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !f.IsDir() {
			totalSize += f.Size()
		}
		return nil
	})
	if err != nil {
		log.Log.Reason(err).Warning("failed to get VMI ephemeral disks size")
		return resource.NewScaledQuantity(0, 0)
	}

	return resource.NewScaledQuantity(totalSize, 0)
}

func getVMIMigrationDataSize(vmi *v1.VirtualMachineInstance) int64 {
	var memory resource.Quantity

	// Take memory from the requested memory
	if v, ok := vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceMemory]; ok {
		memory = v
	}
	// In case that guest memory is explicitly set, override it
	if vmi.Spec.Domain.Memory != nil && vmi.Spec.Domain.Memory.Guest != nil {
		memory = *vmi.Spec.Domain.Memory.Guest
	}

	//get total data Size
	if vmi.Status.MigrationMethod == v1.BlockMigration {
		disksSize := getVMIEphemeralDisksTotalSize()
		memory.Add(*disksSize)
	}
	return memory.ScaledValue(resource.Giga)
}

func (l *LibvirtDomainManager) CancelVMIMigration(vmi *v1.VirtualMachineInstance) error {
	return l.cancelMigration(vmi)
}

func (l *LibvirtDomainManager) MigrateVMI(vmi *v1.VirtualMachineInstance, options *cmdclient.MigrationOptions) error {
	return l.startMigration(vmi, options)
}

func (l *LibvirtDomainManager) generateSomeCloudInitISO(vmi *v1.VirtualMachineInstance, domPtr *cli.VirDomain, size int64) error {
	var devicesMetadata []cloudinit.DeviceData
	// this is the point where we need to build the devices metadata if it was requested.
	// This metadata maps the user provided tag to the hypervisor assigned device address.
	if domPtr != nil {
		data, err := l.buildDevicesMetadata(vmi, *domPtr)
		if err != nil {
			return err
		}
		devicesMetadata = data
	}
	// build condif drive iso file, that includes devices metadata if available
	// get stored cloud init data
	var cloudInitDataStore *cloudinit.CloudInitData
	cloudInitDataStore = l.cloudInitDataStore

	if cloudInitDataStore != nil {
		// add devices metedata
		if devicesMetadata != nil {
			cloudInitDataStore.DevicesData = &devicesMetadata
		}
		var err error
		if size != 0 {
			err = cloudinit.GenerateEmptyIso(vmi.Name, vmi.Namespace, cloudInitDataStore, size)
		} else {
			// ClusterFlavor will take precedence over a namespaced Flavor
			// for setting instance_type in the metadata
			flavor := vmi.Annotations[v1.ClusterFlavorAnnotation]
			if flavor == "" {
				flavor = vmi.Annotations[v1.FlavorAnnotation]
			}

			err = cloudinit.GenerateLocalData(vmi.Name, vmi.Namespace, flavor, cloudInitDataStore)
		}
		if err != nil {
			return fmt.Errorf("generating local cloud-init data failed: %v", err)
		}
	}
	return nil
}

func (l *LibvirtDomainManager) generateCloudInitISO(vmi *v1.VirtualMachineInstance, domPtr *cli.VirDomain) error {
	return l.generateSomeCloudInitISO(vmi, domPtr, 0)
}

func (l *LibvirtDomainManager) generateCloudInitEmptyISO(vmi *v1.VirtualMachineInstance, domPtr *cli.VirDomain) error {
	if l.cloudInitDataStore == nil {
		return nil
	}
	for _, vs := range vmi.Status.VolumeStatus {
		if vs.Name == l.cloudInitDataStore.VolumeName {
			return l.generateSomeCloudInitISO(vmi, domPtr, vs.Size)
		}
	}
	return fmt.Errorf("failed to find the status of volume %s", l.cloudInitDataStore.VolumeName)
}

// All local environment setup that needs to occur before VirtualMachineInstance starts
// can be done in this function. This includes things like...
//
// - storage prep
// - network prep
// - cloud-init
// - sysprep
//
// The Domain.Spec can be alterned in this function and any changes
// made to the domain will get set in libvirt after this function exits.
func (l *LibvirtDomainManager) preStartHook(vmi *v1.VirtualMachineInstance, domain *api.Domain, generateEmptyIsos bool) (*api.Domain, error) {
	logger := log.Log.Object(vmi)

	logger.Info("Executing PreStartHook on VMI pod environment")

	disksInfo := map[string]*containerdisk.DiskInfo{}
	for k, v := range l.disksInfo {
		if v != nil {
			disksInfo[k] = &containerdisk.DiskInfo{
				Format:      v.Format,
				BackingFile: v.BackingFile,
				ActualSize:  int64(v.ActualSize),
				VirtualSize: int64(v.VirtualSize),
			}
		}
	}

	// generate cloud-init data
	cloudInitData, err := cloudinit.ReadCloudInitVolumeDataSource(vmi, config.SecretSourceDir)
	if err != nil {
		return domain, fmt.Errorf("ReadCloudInitVolumeDataSource failed: %v", err)
	}

	// Pass cloud-init data to PreCloudInitIso hook
	logger.Info("Starting PreCloudInitIso hook")
	hooksManager := hooks.GetManager()
	cloudInitData, err = hooksManager.PreCloudInitIso(vmi, cloudInitData)
	if err != nil {
		return domain, fmt.Errorf("PreCloudInitIso hook failed: %v", err)
	}

	if cloudInitData != nil {
		// need to prepare the local path for cloud-init in advance for proper
		// detection of the disk driver cache mode
		if err := cloudinit.PrepareLocalPath(vmi.Name, vmi.Namespace); err != nil {
			return domain, fmt.Errorf("PrepareLocalPath failed: %v", err)
		}
		// store the generated cloud init metadata.
		// cloud init ISO will be generated after the domain definition
		l.cloudInitDataStore = cloudInitData
	}

	// generate ignition data
	ignitionData := ignition.GetIgnitionSource(vmi)
	if ignitionData != "" {

		err := ignition.GenerateIgnitionLocalData(vmi, vmi.Namespace)
		if err != nil {
			return domain, err
		}
	}

	err = netsetup.NewVMNetworkConfigurator(vmi, cache.CacheCreator{}).SetupPodNetworkPhase2(domain)
	if err != nil {
		return domain, fmt.Errorf("preparing the pod network failed: %v", err)
	}

	// Create ephemeral disk for container disks
	err = containerdisk.CreateEphemeralImages(vmi, l.ephemeralDiskCreator, disksInfo)
	if err != nil {
		return domain, fmt.Errorf("preparing ephemeral container disk images failed: %v", err)
	}
	// Create images for volumes that are marked ephemeral.
	err = l.ephemeralDiskCreator.CreateEphemeralImages(vmi, domain)
	if err != nil {
		return domain, fmt.Errorf("preparing ephemeral images failed: %v", err)
	}
	// create empty disks if they exist
	if err := emptydisk.NewEmptyDiskCreator().CreateTemporaryDisks(vmi); err != nil {
		return domain, fmt.Errorf("creating empty disks failed: %v", err)
	}
	// create ConfigMap disks if they exists
	if err := config.CreateConfigMapDisks(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating config map disks failed: %v", err)
	}
	// create Secret disks if they exists
	if err := config.CreateSecretDisks(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating secret disks failed: %v", err)
	}

	// create Sysprep disks if they exists
	if err := config.CreateSysprepDisks(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating sysprep disks failed: %v", err)
	}

	// create DownwardAPI disks if they exists
	if err := config.CreateDownwardAPIDisks(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating DownwardAPI disks failed: %v", err)
	}
	// create ServiceAccount disk if exists
	if err := config.CreateServiceAccountDisk(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating service account disk failed: %v", err)
	}
	// create downwardMetric disk if exists
	if err := downwardmetrics.CreateDownwardMetricDisk(vmi); err != nil {
		return domain, fmt.Errorf("failed to craete downwardMetric disk: %v", err)
	}

	// set drivers cache mode
	for i := range domain.Spec.Devices.Disks {
		err := converter.SetDriverCacheMode(&domain.Spec.Devices.Disks[i], l.directIOChecker)
		if err != nil {
			return domain, err
		}
		converter.SetOptimalIOMode(&domain.Spec.Devices.Disks[i])
	}

	if err := l.credManager.HandleQemuAgentAccessCredentials(vmi); err != nil {
		return domain, fmt.Errorf("Starting qemu agent access credential propagation failed: %v", err)
	}

	// expand disk image files if they're too small
	expandDiskImagesOffline(vmi, domain)

	return domain, err
}

func expandDiskImagesOffline(vmi *v1.VirtualMachineInstance, domain *api.Domain) {
	logger := log.Log.Object(vmi)
	for _, disk := range domain.Spec.Devices.Disks {
		if shouldExpandOffline(disk) {
			possibleGuestSize, ok := possibleGuestSize(disk)
			if !ok {
				logger.Errorf("Failed to get possible guest size from disk")
				break
			}
			err := expandDiskImageOffline(getSourceFile(disk), possibleGuestSize)
			if err != nil {
				logger.Reason(err).Errorf("failed to expand disk image %v at boot", disk)
			}
		}
	}
}

func expandDiskImageOffline(imagePath string, size int64) error {
	log.Log.Infof("pre-start expansion of image %s to size %d", imagePath, size)
	var preallocateFlag string
	if converter.IsPreAllocated(imagePath) {
		preallocateFlag = "--preallocation=falloc"
	} else {
		preallocateFlag = "--preallocation=off"
	}
	size = kutil.AlignImageSizeTo1MiB(size, log.Log.With("image", imagePath))
	if size == 0 {
		return fmt.Errorf("%s must be at least 1MiB", imagePath)
	}
	cmd := exec.Command("/usr/bin/qemu-img", "resize", preallocateFlag, imagePath, strconv.FormatInt(size, 10))
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("expanding image failed with error: %v, output: %s", err, out)
	}
	return nil
}

func possibleGuestSize(disk api.Disk) (int64, bool) {
	if disk.Capacity == nil {
		log.DefaultLogger().Error("No disk capacity")
		return 0, false
	}
	preferredSize := *disk.Capacity
	if disk.FilesystemOverhead == nil {
		log.DefaultLogger().Errorf("No filesystem overhead found for disk %v", disk)
		return 0, false
	}
	filesystemOverhead, err := strconv.ParseFloat(string(*disk.FilesystemOverhead), 64)
	if err != nil {
		log.DefaultLogger().Reason(err).Error("Failed to parse filesystem overhead as float")
		return 0, false
	}
	size := int64((1 - filesystemOverhead) * float64(preferredSize))
	size = kutil.AlignImageSizeTo1MiB(size, log.DefaultLogger())
	if size == 0 {
		return 0, false
	}
	return size, true
}

func shouldExpandOffline(disk api.Disk) bool {
	if !disk.ExpandDisksEnabled {
		return false
	}
	if disk.Source.Dev != "" {
		// Block devices don't need to be expanded
		return false
	}
	diskInfo, err := converter.GetImageInfo(getSourceFile(disk))
	if err != nil {
		log.DefaultLogger().Reason(err).Warning("Failed to get image info")
		return false
	}
	possibleGuestSize, ok := possibleGuestSize(disk)
	if !ok || possibleGuestSize <= diskInfo.VirtualSize {
		return false
	}
	return true
}

func (l *LibvirtDomainManager) generateConverterContext(vmi *v1.VirtualMachineInstance, allowEmulation bool, options *cmdv1.VirtualMachineOptions, isMigrationTarget bool) (*converter.ConverterContext, error) {

	logger := log.Log.Object(vmi)

	podCPUSet, err := util.GetPodCPUSet()
	if err != nil {
		logger.Reason(err).Error("failed to read pod cpuset.")
		return nil, fmt.Errorf("failed to read pod cpuset: %v", err)
	}

	hotplugVolumes := make(map[string]v1.VolumeStatus)
	permanentVolumes := make(map[string]v1.VolumeStatus)
	for _, status := range vmi.Status.VolumeStatus {
		if status.HotplugVolume != nil {
			hotplugVolumes[status.Name] = status
		} else {
			permanentVolumes[status.Name] = status
		}
	}

	// Check if PVC volumes are block volumes
	isBlockPVCMap := make(map[string]bool)
	isBlockDVMap := make(map[string]bool)
	for _, volume := range vmi.Spec.Volumes {
		if volume.VolumeSource.PersistentVolumeClaim != nil || volume.VolumeSource.Ephemeral != nil {
			isBlockPVC := false
			if _, ok := hotplugVolumes[volume.Name]; ok {
				isBlockPVC = isHotplugBlockDeviceVolume(volume.Name)
			} else {
				isBlockPVC, _ = isBlockDeviceVolume(volume.Name)
			}
			isBlockPVCMap[volume.Name] = isBlockPVC
		} else if volume.VolumeSource.DataVolume != nil {
			isBlockDV := false
			if _, ok := hotplugVolumes[volume.Name]; ok {
				isBlockDV = isHotplugBlockDeviceVolume(volume.Name)
			} else {
				isBlockDV, _ = isBlockDeviceVolume(volume.Name)
			}
			isBlockDVMap[volume.Name] = isBlockDV
		}
	}

	var efiConf *converter.EFIConfiguration
	if vmi.IsBootloaderEFI() {
		secureBoot := vmi.Spec.Domain.Firmware.Bootloader.EFI.SecureBoot == nil || *vmi.Spec.Domain.Firmware.Bootloader.EFI.SecureBoot
		sev := kutil.IsSEVVMI(vmi)

		if !l.efiEnvironment.Bootable(secureBoot, sev) {
			log.Log.Reason(err).Errorf("EFI OVMF roms missing for booting in EFI mode with SecureBoot=%v, SEV=%v", secureBoot, sev)
			return nil, fmt.Errorf("EFI OVMF roms missing for booting in EFI mode with SecureBoot=%v, SEV=%v", secureBoot, sev)
		}

		efiConf = &converter.EFIConfiguration{
			EFICode:      l.efiEnvironment.EFICode(secureBoot, sev),
			EFIVars:      l.efiEnvironment.EFIVars(secureBoot, sev),
			SecureLoader: secureBoot,
		}
	}

	// Map the VirtualMachineInstance to the Domain
	c := &converter.ConverterContext{
		Architecture:          runtime.GOARCH,
		VirtualMachine:        vmi,
		AllowEmulation:        allowEmulation,
		CPUSet:                podCPUSet,
		IsBlockPVC:            isBlockPVCMap,
		IsBlockDV:             isBlockDVMap,
		EFIConfiguration:      efiConf,
		UseVirtioTransitional: vmi.Spec.Domain.Devices.UseVirtioTransitional != nil && *vmi.Spec.Domain.Devices.UseVirtioTransitional,
		PermanentVolumes:      permanentVolumes,
		EphemeraldiskCreator:  l.ephemeralDiskCreator,
		UseLaunchSecurity:     kutil.IsSEVVMI(vmi),
	}

	if options != nil {
		c.ExpandDisksEnabled = options.ExpandDisksEnabled
		if options.VirtualMachineSMBios != nil {
			c.SMBios = options.VirtualMachineSMBios
		}
		if options.Topology != nil {
			c.Topology = options.Topology
		}
		c.MemBalloonStatsPeriod = uint(options.MemBalloonStatsPeriod)
		// Add preallocated and thick-provisioned volumes for which we need to avoid the discard=unmap option
		c.VolumesDiscardIgnore = options.PreallocatedVolumes

		if len(options.DisksInfo) > 0 {
			l.disksInfo = options.DisksInfo
		}
	}
	c.DisksInfo = l.disksInfo

	if !isMigrationTarget {
		sriovDevices, err := sriov.CreateHostDevices(vmi)
		if err != nil {
			return nil, err
		}

		c.HotplugVolumes = hotplugVolumes
		c.SRIOVDevices = sriovDevices

		genericHostDevices, err := generic.CreateHostDevices(vmi.Spec.Domain.Devices.HostDevices)
		if err != nil {
			return nil, err
		}
		c.GenericHostDevices = genericHostDevices

		gpuHostDevices, err := gpu.CreateHostDevices(vmi.Spec.Domain.Devices.GPUs)
		if err != nil {
			return nil, err
		}
		c.GPUHostDevices = gpuHostDevices
	}

	return c, nil
}

func (l *LibvirtDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, allowEmulation bool, options *cmdv1.VirtualMachineOptions) (*api.DomainSpec, error) {
	l.domainModifyLock.Lock()
	defer l.domainModifyLock.Unlock()

	logger := log.Log.Object(vmi)

	domain := &api.Domain{}

	c, err := l.generateConverterContext(vmi, allowEmulation, options, false)
	if err != nil {
		logger.Reason(err).Error("failed to generate libvirt domain from VMI spec")
		return nil, err
	}

	if err := converter.Convert_v1_VirtualMachineInstance_To_api_Domain(vmi, domain, c); err != nil {
		logger.Error("Conversion failed.")
		return nil, err
	}

	// Set defaults which are not coming from the cluster
	api.NewDefaulter(c.Architecture).SetObjectDefaults_Domain(domain)

	dom, err := l.virConn.LookupDomainByName(domain.Spec.Name)
	if err != nil {
		// We need the domain but it does not exist, so create it
		if domainerrors.IsNotFound(err) {
			domain, err = l.preStartHook(vmi, domain, false)
			if err != nil {
				logger.Reason(err).Error("pre start setup for VirtualMachineInstance failed.")
				return nil, err
			}
			dom, err = l.setDomainSpecWithHooks(vmi, &domain.Spec)
			if err != nil {
				return nil, err
			}
			logger.Info("Domain defined.")
		} else {
			logger.Reason(err).Error(failedGetDomain)
			return nil, err
		}
	}
	defer dom.Free()
	domState, _, err := dom.GetState()
	if err != nil {
		logger.Reason(err).Error(failedGetDomainState)
		return nil, err
	}

	// TODO Suspend, Pause, ..., for now we only support reaching the running state
	// TODO for migration and error detection we also need the state change reason
	// TODO blocked state
	if cli.IsDown(domState) && !vmi.IsRunning() && !vmi.IsFinal() {
		err = l.generateCloudInitISO(vmi, &dom)
		if err != nil {
			return nil, err
		}
		createFlags := getDomainCreateFlags(vmi)
		err = dom.CreateWithFlags(createFlags)
		if err != nil {
			logger.Reason(err).
				Errorf("Failed to start VirtualMachineInstance with flags %v.", createFlags)
			return nil, err
		}
		logger.Info("Domain started.")
		if vmi.ShouldStartPaused() {
			l.paused.add(vmi.UID)
		}
	} else if cli.IsPaused(domState) && !l.paused.contains(vmi.UID) {
		// TODO: if state change reason indicates a system error, we could try something smarter
		err := dom.Resume()
		if err != nil {
			logger.Reason(err).Error("unpausing the VirtualMachineInstance failed.")
			return nil, err
		}
		logger.Info("Domain unpaused.")
	} else {
		// Nothing to do
	}

	xmlstr, err := dom.GetXMLDesc(0)
	if err != nil {
		return nil, err
	}

	var oldSpec api.DomainSpec
	err = xml.Unmarshal([]byte(xmlstr), &oldSpec)
	if err != nil {
		logger.Reason(err).Error("Parsing domain XML failed.")
		return nil, err
	}

	//Look up all the disks to detach
	for _, detachDisk := range getDetachedDisks(oldSpec.Devices.Disks, domain.Spec.Devices.Disks) {
		logger.V(1).Infof("Detaching disk %s, target %s", detachDisk.Alias.GetName(), detachDisk.Target.Device)
		detachBytes, err := xml.Marshal(detachDisk)
		if err != nil {
			logger.Reason(err).Error("marshalling detached disk failed")
			return nil, err
		}
		err = dom.DetachDevice(strings.ToLower(string(detachBytes)))
		if err != nil {
			logger.Reason(err).Error("detaching device")
			return nil, err
		}
	}
	//Look up all the disks to attach
	for _, attachDisk := range getAttachedDisks(oldSpec.Devices.Disks, domain.Spec.Devices.Disks) {
		allowAttach, err := checkIfDiskReadyToUse(getSourceFile(attachDisk))
		if err != nil {
			return nil, err
		}
		if !allowAttach {
			continue
		}
		logger.V(1).Infof("Attaching disk %s, target %s", attachDisk.Alias.GetName(), attachDisk.Target.Device)
		attachBytes, err := xml.Marshal(attachDisk)
		if err != nil {
			logger.Reason(err).Error("marshalling attached disk failed")
			return nil, err<commit_msg>Remove incorrect reason when logging<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017, 2018 Red Hat, Inc.
 *
 */

package virtwrap

//go:generate mockgen -source $GOFILE -package=$GOPACKAGE -destination=generated_mock_$GOFILE

/*
 ATTENTION: Rerun code generators when interface signatures are modified.
*/

import (
	"context"
	"encoding/xml"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/generic"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/gpu"

	"kubevirt.io/kubevirt/pkg/downwardmetrics"
	"kubevirt.io/kubevirt/pkg/network/cache"
	"kubevirt.io/kubevirt/pkg/util/hardware"
	cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/agent"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/converter"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/efi"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"libvirt.org/go/libvirt"

	v1 "kubevirt.io/api/core/v1"
	"kubevirt.io/client-go/log"
	cloudinit "kubevirt.io/kubevirt/pkg/cloud-init"
	"kubevirt.io/kubevirt/pkg/config"
	containerdisk "kubevirt.io/kubevirt/pkg/container-disk"
	"kubevirt.io/kubevirt/pkg/controller"
	"kubevirt.io/kubevirt/pkg/emptydisk"
	ephemeraldisk "kubevirt.io/kubevirt/pkg/ephemeral-disk"
	cmdv1 "kubevirt.io/kubevirt/pkg/handler-launcher-com/cmd/v1"
	"kubevirt.io/kubevirt/pkg/hooks"
	"kubevirt.io/kubevirt/pkg/ignition"
	netsetup "kubevirt.io/kubevirt/pkg/network/setup"
	netsriov "kubevirt.io/kubevirt/pkg/network/sriov"
	kutil "kubevirt.io/kubevirt/pkg/util"
	accesscredentials "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/access-credentials"
	agentpoller "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/agent-poller"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/cli"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/sriov"
	domainerrors "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/errors"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/util"
)

const (
	failedSyncGuestTime  = "failed to sync guest time"
	failedGetDomain      = "Getting the domain failed."
	failedGetDomainState = "Getting the domain state failed."
)

const (
	PCI_RESOURCE_PREFIX  = "PCI_RESOURCE"
	MDEV_RESOURCE_PREFIX = "MDEV_PCI_RESOURCE"

	baseEphemeralDiskDir = "/var/run/kubevirt-ephemeral-disks/"
)

const maxConcurrentHotplugHostDevices = 1

var (
	getEphemeralDiskBaseDir = func() string {
		return baseEphemeralDiskDir
	}
)

type contextStore struct {
	ctx    context.Context
	cancel context.CancelFunc
}

type DomainManager interface {
	SyncVMI(*v1.VirtualMachineInstance, bool, *cmdv1.VirtualMachineOptions) (*api.DomainSpec, error)
	PauseVMI(*v1.VirtualMachineInstance) error
	UnpauseVMI(*v1.VirtualMachineInstance) error
	FreezeVMI(*v1.VirtualMachineInstance, int32) error
	UnfreezeVMI(*v1.VirtualMachineInstance) error
	SoftRebootVMI(*v1.VirtualMachineInstance) error
	KillVMI(*v1.VirtualMachineInstance) error
	DeleteVMI(*v1.VirtualMachineInstance) error
	SignalShutdownVMI(*v1.VirtualMachineInstance) error
	MarkGracefulShutdownVMI(*v1.VirtualMachineInstance) error
	ListAllDomains() ([]*api.Domain, error)
	MigrateVMI(*v1.VirtualMachineInstance, *cmdclient.MigrationOptions) error
	PrepareMigrationTarget(*v1.VirtualMachineInstance, bool, *cmdv1.VirtualMachineOptions) error
	GetDomainStats() ([]*stats.DomainStats, error)
	CancelVMIMigration(*v1.VirtualMachineInstance) error
	GetGuestInfo() (v1.VirtualMachineInstanceGuestAgentInfo, error)
	GetUsers() ([]v1.VirtualMachineInstanceGuestOSUser, error)
	GetFilesystems() ([]v1.VirtualMachineInstanceFileSystem, error)
	FinalizeVirtualMachineMigration(*v1.VirtualMachineInstance) error
	HotplugHostDevices(vmi *v1.VirtualMachineInstance) error
	InterfacesStatus() []api.InterfaceStatus
	GetGuestOSInfo() *api.GuestOSInfo
	Exec(string, string, []string, int32) (string, error)
	GuestPing(string) error
}

type LibvirtDomainManager struct {
	virConn cli.Connection

	// Anytime a get and a set is done on the domain, this lock must be held.
	domainModifyLock sync.Mutex
	// mutex to control access to the guest time context
	setGuestTimeLock sync.Mutex

	credManager *accesscredentials.AccessCredentialManager

	hotplugHostDevicesInProgress chan struct{}

	virtShareDir             string
	paused                   pausedVMIs
	agentData                *agentpoller.AsyncAgentStore
	cloudInitDataStore       *cloudinit.CloudInitData
	setGuestTimeContextPtr   *contextStore
	efiEnvironment           *efi.EFIEnvironment
	ovmfPath                 string
	ephemeralDiskCreator     ephemeraldisk.EphemeralDiskCreatorInterface
	directIOChecker          converter.DirectIOChecker
	disksInfo                map[string]*cmdv1.DiskInfo
	cancelSafetyUnfreezeChan chan struct{}
}

type pausedVMIs struct {
	paused map[types.UID]bool
}

func (s pausedVMIs) add(uid types.UID) {
	// implicitly locked by domainModifyLock
	if _, ok := s.paused[uid]; !ok {
		s.paused[uid] = true
	}
}

func (s pausedVMIs) remove(uid types.UID) {
	// implicitly locked by domainModifyLock
	if _, ok := s.paused[uid]; ok {
		delete(s.paused, uid)
	}
}

func (s pausedVMIs) contains(uid types.UID) bool {
	_, ok := s.paused[uid]
	return ok
}

func NewLibvirtDomainManager(connection cli.Connection, virtShareDir string, agentStore *agentpoller.AsyncAgentStore, ovmfPath string, ephemeralDiskCreator ephemeraldisk.EphemeralDiskCreatorInterface) (DomainManager, error) {
	directIOChecker := converter.NewDirectIOChecker()
	return newLibvirtDomainManager(connection, virtShareDir, agentStore, ovmfPath, ephemeralDiskCreator, directIOChecker)
}

func newLibvirtDomainManager(connection cli.Connection, virtShareDir string, agentStore *agentpoller.AsyncAgentStore, ovmfPath string, ephemeralDiskCreator ephemeraldisk.EphemeralDiskCreatorInterface, directIOChecker converter.DirectIOChecker) (DomainManager, error) {
	manager := LibvirtDomainManager{
		virConn:      connection,
		virtShareDir: virtShareDir,
		paused: pausedVMIs{
			paused: make(map[types.UID]bool, 0),
		},
		agentData:                agentStore,
		efiEnvironment:           efi.DetectEFIEnvironment(runtime.GOARCH, ovmfPath),
		ephemeralDiskCreator:     ephemeralDiskCreator,
		directIOChecker:          directIOChecker,
		disksInfo:                map[string]*cmdv1.DiskInfo{},
		cancelSafetyUnfreezeChan: make(chan struct{}),
	}

	manager.hotplugHostDevicesInProgress = make(chan struct{}, maxConcurrentHotplugHostDevices)
	manager.credManager = accesscredentials.NewManager(connection, &manager.domainModifyLock)

	return &manager, nil
}

func getDomainSpec(dom cli.VirDomain) (*api.DomainSpec, error) {
	var newSpec api.DomainSpec
	xmlstr, err := dom.GetXMLDesc(0)
	if err != nil {
		return &newSpec, err
	}
	err = xml.Unmarshal([]byte(xmlstr), &newSpec)
	if err != nil {
		return &newSpec, err
	}

	return &newSpec, nil
}

func getAllDomainDevices(dom cli.VirDomain) (api.Devices, error) {
	domSpec, err := getDomainSpec(dom)
	if err != nil {
		return domSpec.Devices, err
	}
	return domSpec.Devices, nil
}

func getAllDomainDisks(dom cli.VirDomain) ([]api.Disk, error) {
	devices, err := getAllDomainDevices(dom)
	if err != nil {
		return nil, err
	}

	return devices.Disks, nil
}

func (l *LibvirtDomainManager) setGuestTime(vmi *v1.VirtualMachineInstance) error {
	// Try to set VM time to the current value.  This is typically useful
	// when clock wasn't running on the VM for some time (e.g. during
	// suspension or migration), especially if the time delay exceeds NTP
	// tolerance.
	// It is not guaranteed that the time is actually set (it depends on guest
	// environment, especially QEMU agent presence) or that the set time is
	// very precise (NTP in the guest should take care of it if needed).

	domName := api.VMINamespaceKeyFunc(vmi)
	dom, err := l.virConn.LookupDomainByName(domName)
	if err != nil {
		log.Log.Object(vmi).Reason(err).Error(failedSyncGuestTime)
		return err
	}

	go func() {
		defer dom.Free()

		ctx := l.getGuestTimeContext()
		timeout := time.After(60 * time.Second)
		ticker := time.NewTicker(time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-timeout:
				log.Log.Object(vmi).Error(failedSyncGuestTime)
				return
			case <-ctx.Done():
				return
			case <-ticker.C:
				currTime := time.Now()
				secs := currTime.Unix()
				nsecs := uint(currTime.Nanosecond())
				err := dom.SetTime(secs, nsecs, 0)
				if err != nil {
					libvirtError, ok := err.(libvirt.Error)
					if !ok {
						log.Log.Object(vmi).Reason(err).Warning(failedSyncGuestTime)
						return
					}

					switch libvirtError.Code {
					case libvirt.ERR_AGENT_UNRESPONSIVE:
						log.Log.Object(vmi).Reason(err).Warning("failed to set time: QEMU agent unresponsive")
					case libvirt.ERR_OPERATION_UNSUPPORTED:
						// no need to retry as this opertaion is not supported
						log.Log.Object(vmi).Reason(err).Warning("failed to set time: not supported")
						return
					case libvirt.ERR_ARGUMENT_UNSUPPORTED:
						// no need to retry as the agent is not configured
						log.Log.Object(vmi).Reason(err).Warning("failed to set time: agent not configured")
						return
					default:
						log.Log.Object(vmi).Reason(err).Warning(failedSyncGuestTime)
					}
				} else {
					log.Log.Object(vmi).Info("guest VM time sync finished successfully")
					return
				}
			}
		}
	}()

	return nil
}

func (l *LibvirtDomainManager) getGuestTimeContext() context.Context {
	l.setGuestTimeLock.Lock()
	defer l.setGuestTimeLock.Unlock()

	// cancel the already running setGuestTime go-routine if such exist
	if l.setGuestTimeContextPtr != nil {
		l.setGuestTimeContextPtr.cancel()
	}
	// create a new context and store it
	ctx, cancel := context.WithCancel(context.Background())
	l.setGuestTimeContextPtr = &contextStore{ctx: ctx, cancel: cancel}
	return ctx
}

// PrepareMigrationTarget the target pod environment before the migration is initiated
func (l *LibvirtDomainManager) PrepareMigrationTarget(
	vmi *v1.VirtualMachineInstance,
	allowEmulation bool,
	options *cmdv1.VirtualMachineOptions,
) error {
	return l.prepareMigrationTarget(vmi, allowEmulation, options)
}

// FinalizeVirtualMachineMigration finalized the migration after the migration has completed and vmi is running on target pod.
func (l *LibvirtDomainManager) FinalizeVirtualMachineMigration(vmi *v1.VirtualMachineInstance) error {
	return l.finalizeMigrationTarget(vmi)
}

// HotplugHostDevices attach host-devices to running domain, currently only SRIOV host-devices are supported.
// This operation runs in the background, only one hotplug operation can occur at a time.
func (l *LibvirtDomainManager) HotplugHostDevices(vmi *v1.VirtualMachineInstance) error {
	select {
	case l.hotplugHostDevicesInProgress <- struct{}{}:
	default:
		return fmt.Errorf("hot-plug host-devices is in progress")
	}

	go func() {
		defer func() { <-l.hotplugHostDevicesInProgress }()

		if err := l.hotPlugHostDevices(vmi); err != nil {
			log.Log.Object(vmi).Error(err.Error())
		}
	}()
	return nil
}

func (l *LibvirtDomainManager) hotPlugHostDevices(vmi *v1.VirtualMachineInstance) error {
	l.domainModifyLock.Lock()
	defer l.domainModifyLock.Unlock()

	const errMsgPrefix = "failed to hot-plug host-devices"

	domainName := api.VMINamespaceKeyFunc(vmi)
	domain, err := l.virConn.LookupDomainByName(domainName)
	if err != nil {
		return fmt.Errorf("%s: %v", errMsgPrefix, err)
	}
	defer domain.Free()

	domainSpec, err := util.GetDomainSpecWithFlags(domain, 0)
	if err != nil {
		return fmt.Errorf("%s: %v", errMsgPrefix, err)
	}

	sriovHostDevices, err := sriov.GetHostDevicesToAttach(vmi, domainSpec)
	if err != nil {
		return fmt.Errorf("%s: %v", errMsgPrefix, err)
	}

	if err := hostdevice.AttachHostDevices(domain, sriovHostDevices); err != nil {
		return fmt.Errorf("%s: %v", errMsgPrefix, hostdevice.AttachHostDevices(domain, sriovHostDevices))
	}

	return nil
}

func (l *LibvirtDomainManager) Exec(domainName, command string, args []string, timeoutSeconds int32) (string, error) {
	return agent.GuestExec(l.virConn, domainName, command, args, timeoutSeconds)
}

func (l *LibvirtDomainManager) GuestPing(domainName string) error {
	pingCmd := `{"execute":"guest-ping"}`
	_, err := l.virConn.QemuAgentCommand(pingCmd, domainName)
	return err
}

func getVMIEphemeralDisksTotalSize() *resource.Quantity {
	baseDir := getEphemeralDiskBaseDir()
	totalSize := int64(0)
	err := filepath.Walk(baseDir, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !f.IsDir() {
			totalSize += f.Size()
		}
		return nil
	})
	if err != nil {
		log.Log.Reason(err).Warning("failed to get VMI ephemeral disks size")
		return resource.NewScaledQuantity(0, 0)
	}

	return resource.NewScaledQuantity(totalSize, 0)
}

func getVMIMigrationDataSize(vmi *v1.VirtualMachineInstance) int64 {
	var memory resource.Quantity

	// Take memory from the requested memory
	if v, ok := vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceMemory]; ok {
		memory = v
	}
	// In case that guest memory is explicitly set, override it
	if vmi.Spec.Domain.Memory != nil && vmi.Spec.Domain.Memory.Guest != nil {
		memory = *vmi.Spec.Domain.Memory.Guest
	}

	// get total data Size
	if vmi.Status.MigrationMethod == v1.BlockMigration {
		disksSize := getVMIEphemeralDisksTotalSize()
		memory.Add(*disksSize)
	}
	return memory.ScaledValue(resource.Giga)
}

func (l *LibvirtDomainManager) CancelVMIMigration(vmi *v1.VirtualMachineInstance) error {
	return l.cancelMigration(vmi)
}

func (l *LibvirtDomainManager) MigrateVMI(vmi *v1.VirtualMachineInstance, options *cmdclient.MigrationOptions) error {
	return l.startMigration(vmi, options)
}

func (l *LibvirtDomainManager) generateSomeCloudInitISO(vmi *v1.VirtualMachineInstance, domPtr *cli.VirDomain, size int64) error {
	var devicesMetadata []cloudinit.DeviceData
	// this is the point where we need to build the devices metadata if it was requested.
	// This metadata maps the user provided tag to the hypervisor assigned device address.
	if domPtr != nil {
		data, err := l.buildDevicesMetadata(vmi, *domPtr)
		if err != nil {
			return err
		}
		devicesMetadata = data
	}
	// build condif drive iso file, that includes devices metadata if available
	// get stored cloud init data
	var cloudInitDataStore *cloudinit.CloudInitData
	cloudInitDataStore = l.cloudInitDataStore

	if cloudInitDataStore != nil {
		// add devices metedata
		if devicesMetadata != nil {
			cloudInitDataStore.DevicesData = &devicesMetadata
		}
		var err error
		if size != 0 {
			err = cloudinit.GenerateEmptyIso(vmi.Name, vmi.Namespace, cloudInitDataStore, size)
		} else {
			// ClusterFlavor will take precedence over a namespaced Flavor
			// for setting instance_type in the metadata
			flavor := vmi.Annotations[v1.ClusterFlavorAnnotation]
			if flavor == "" {
				flavor = vmi.Annotations[v1.FlavorAnnotation]
			}

			err = cloudinit.GenerateLocalData(vmi.Name, vmi.Namespace, flavor, cloudInitDataStore)
		}
		if err != nil {
			return fmt.Errorf("generating local cloud-init data failed: %v", err)
		}
	}
	return nil
}

func (l *LibvirtDomainManager) generateCloudInitISO(vmi *v1.VirtualMachineInstance, domPtr *cli.VirDomain) error {
	return l.generateSomeCloudInitISO(vmi, domPtr, 0)
}

func (l *LibvirtDomainManager) generateCloudInitEmptyISO(vmi *v1.VirtualMachineInstance, domPtr *cli.VirDomain) error {
	if l.cloudInitDataStore == nil {
		return nil
	}
	for _, vs := range vmi.Status.VolumeStatus {
		if vs.Name == l.cloudInitDataStore.VolumeName {
			return l.generateSomeCloudInitISO(vmi, domPtr, vs.Size)
		}
	}
	return fmt.Errorf("failed to find the status of volume %s", l.cloudInitDataStore.VolumeName)
}

// All local environment setup that needs to occur before VirtualMachineInstance starts
// can be done in this function. This includes things like...
//
// - storage prep
// - network prep
// - cloud-init
// - sysprep
//
// The Domain.Spec can be alterned in this function and any changes
// made to the domain will get set in libvirt after this function exits.
func (l *LibvirtDomainManager) preStartHook(vmi *v1.VirtualMachineInstance, domain *api.Domain, generateEmptyIsos bool) (*api.Domain, error) {
	logger := log.Log.Object(vmi)

	logger.Info("Executing PreStartHook on VMI pod environment")

	disksInfo := map[string]*containerdisk.DiskInfo{}
	for k, v := range l.disksInfo {
		if v != nil {
			disksInfo[k] = &containerdisk.DiskInfo{
				Format:      v.Format,
				BackingFile: v.BackingFile,
				ActualSize:  int64(v.ActualSize),
				VirtualSize: int64(v.VirtualSize),
			}
		}
	}

	// generate cloud-init data
	cloudInitData, err := cloudinit.ReadCloudInitVolumeDataSource(vmi, config.SecretSourceDir)
	if err != nil {
		return domain, fmt.Errorf("ReadCloudInitVolumeDataSource failed: %v", err)
	}

	// Pass cloud-init data to PreCloudInitIso hook
	logger.Info("Starting PreCloudInitIso hook")
	hooksManager := hooks.GetManager()
	cloudInitData, err = hooksManager.PreCloudInitIso(vmi, cloudInitData)
	if err != nil {
		return domain, fmt.Errorf("PreCloudInitIso hook failed: %v", err)
	}

	if cloudInitData != nil {
		// need to prepare the local path for cloud-init in advance for proper
		// detection of the disk driver cache mode
		if err := cloudinit.PrepareLocalPath(vmi.Name, vmi.Namespace); err != nil {
			return domain, fmt.Errorf("PrepareLocalPath failed: %v", err)
		}
		// store the generated cloud init metadata.
		// cloud init ISO will be generated after the domain definition
		l.cloudInitDataStore = cloudInitData
	}

	// generate ignition data
	ignitionData := ignition.GetIgnitionSource(vmi)
	if ignitionData != "" {

		err := ignition.GenerateIgnitionLocalData(vmi, vmi.Namespace)
		if err != nil {
			return domain, err
		}
	}

	err = netsetup.NewVMNetworkConfigurator(vmi, cache.CacheCreator{}).SetupPodNetworkPhase2(domain)
	if err != nil {
		return domain, fmt.Errorf("preparing the pod network failed: %v", err)
	}

	// Create ephemeral disk for container disks
	err = containerdisk.CreateEphemeralImages(vmi, l.ephemeralDiskCreator, disksInfo)
	if err != nil {
		return domain, fmt.Errorf("preparing ephemeral container disk images failed: %v", err)
	}
	// Create images for volumes that are marked ephemeral.
	err = l.ephemeralDiskCreator.CreateEphemeralImages(vmi, domain)
	if err != nil {
		return domain, fmt.Errorf("preparing ephemeral images failed: %v", err)
	}
	// create empty disks if they exist
	if err := emptydisk.NewEmptyDiskCreator().CreateTemporaryDisks(vmi); err != nil {
		return domain, fmt.Errorf("creating empty disks failed: %v", err)
	}
	// create ConfigMap disks if they exists
	if err := config.CreateConfigMapDisks(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating config map disks failed: %v", err)
	}
	// create Secret disks if they exists
	if err := config.CreateSecretDisks(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating secret disks failed: %v", err)
	}

	// create Sysprep disks if they exists
	if err := config.CreateSysprepDisks(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating sysprep disks failed: %v", err)
	}

	// create DownwardAPI disks if they exists
	if err := config.CreateDownwardAPIDisks(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating DownwardAPI disks failed: %v", err)
	}
	// create ServiceAccount disk if exists
	if err := config.CreateServiceAccountDisk(vmi, generateEmptyIsos); err != nil {
		return domain, fmt.Errorf("creating service account disk failed: %v", err)
	}
	// create downwardMetric disk if exists
	if err := downwardmetrics.CreateDownwardMetricDisk(vmi); err != nil {
		return domain, fmt.Errorf("failed to craete downwardMetric disk: %v", err)
	}

	// set drivers cache mode
	for i := range domain.Spec.Devices.Disks {
		err := converter.SetDriverCacheMode(&domain.Spec.Devices.Disks[i], l.directIOChecker)
		if err != nil {
			return domain, err
		}
		converter.SetOptimalIOMode(&domain.Spec.Devices.Disks[i])
	}

	if err := l.credManager.HandleQemuAgentAccessCredentials(vmi); err != nil {
		return domain, fmt.Errorf("Starting qemu agent access credential propagation failed: %v", err)
	}

	// expand disk image files if they're too small
	expandDiskImagesOffline(vmi, domain)

	return domain, err
}

func expandDiskImagesOffline(vmi *v1.VirtualMachineInstance, domain *api.Domain) {
	logger := log.Log.Object(vmi)
	for _, disk := range domain.Spec.Devices.Disks {
		if shouldExpandOffline(disk) {
			possibleGuestSize, ok := possibleGuestSize(disk)
			if !ok {
				logger.Errorf("Failed to get possible guest size from disk")
				break
			}
			err := expandDiskImageOffline(getSourceFile(disk), possibleGuestSize)
			if err != nil {
				logger.Reason(err).Errorf("failed to expand disk image %v at boot", disk)
			}
		}
	}
}

func expandDiskImageOffline(imagePath string, size int64) error {
	log.Log.Infof("pre-start expansion of image %s to size %d", imagePath, size)
	var preallocateFlag string
	if converter.IsPreAllocated(imagePath) {
		preallocateFlag = "--preallocation=falloc"
	} else {
		preallocateFlag = "--preallocation=off"
	}
	size = kutil.AlignImageSizeTo1MiB(size, log.Log.With("image", imagePath))
	if size == 0 {
		return fmt.Errorf("%s must be at least 1MiB", imagePath)
	}
	cmd := exec.Command("/usr/bin/qemu-img", "resize", preallocateFlag, imagePath, strconv.FormatInt(size, 10))
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("expanding image failed with error: %v, output: %s", err, out)
	}
	return nil
}

func possibleGuestSize(disk api.Disk) (int64, bool) {
	if disk.Capacity == nil {
		log.DefaultLogger().Error("No disk capacity")
		return 0, false
	}
	preferredSize := *disk.Capacity
	if disk.FilesystemOverhead == nil {
		log.DefaultLogger().Errorf("No filesystem overhead found for disk %v", disk)
		return 0, false
	}
	filesystemOverhead, err := strconv.ParseFloat(string(*disk.FilesystemOverhead), 64)
	if err != nil {
		log.DefaultLogger().Reason(err).Error("Failed to parse filesystem overhead as float")
		return 0, false
	}
	size := int64((1 - filesystemOverhead) * float64(preferredSize))
	size = kutil.AlignImageSizeTo1MiB(size, log.DefaultLogger())
	if size == 0 {
		return 0, false
	}
	return size, true
}

func shouldExpandOffline(disk api.Disk) bool {
	if !disk.ExpandDisksEnabled {
		return false
	}
	if disk.Source.Dev != "" {
		// Block devices don't need to be expanded
		return false
	}
	diskInfo, err := converter.GetImageInfo(getSourceFile(disk))
	if err != nil {
		log.DefaultLogger().Reason(err).Warning("Failed to get image info")
		return false
	}
	possibleGuestSize, ok := possibleGuestSize(disk)
	if !ok || possibleGuestSize <= diskInfo.VirtualSize {
		return false
	}
	return true
}

func (l *LibvirtDomainManager) generateConverterContext(vmi *v1.VirtualMachineInstance, allowEmulation bool, options *cmdv1.VirtualMachineOptions, isMigrationTarget bool) (*converter.ConverterContext, error) {

	logger := log.Log.Object(vmi)

	podCPUSet, err := util.GetPodCPUSet()
	if err != nil {
		logger.Reason(err).Error("failed to read pod cpuset.")
		return nil, fmt.Errorf("failed to read pod cpuset: %v", err)
	}

	hotplugVolumes := make(map[string]v1.VolumeStatus)
	permanentVolumes := make(map[string]v1.VolumeStatus)
	for _, status := range vmi.Status.VolumeStatus {
		if status.HotplugVolume != nil {
			hotplugVolumes[status.Name] = status
		} else {
			permanentVolumes[status.Name] = status
		}
	}

	// Check if PVC volumes are block volumes
	isBlockPVCMap := make(map[string]bool)
	isBlockDVMap := make(map[string]bool)
	for _, volume := range vmi.Spec.Volumes {
		if volume.VolumeSource.PersistentVolumeClaim != nil || volume.VolumeSource.Ephemeral != nil {
			isBlockPVC := false
			if _, ok := hotplugVolumes[volume.Name]; ok {
				isBlockPVC = isHotplugBlockDeviceVolume(volume.Name)
			} else {
				isBlockPVC, _ = isBlockDeviceVolume(volume.Name)
			}
			isBlockPVCMap[volume.Name] = isBlockPVC
		} else if volume.VolumeSource.DataVolume != nil {
			isBlockDV := false
			if _, ok := hotplugVolumes[volume.Name]; ok {
				isBlockDV = isHotplugBlockDeviceVolume(volume.Name)
			} else {
				isBlockDV, _ = isBlockDeviceVolume(volume.Name)
			}
			isBlockDVMap[volume.Name] = isBlockDV
		}
	}

	var efiConf *converter.EFIConfiguration
	if vmi.IsBootloaderEFI() {
		secureBoot := vmi.Spec.Domain.Firmware.Bootloader.EFI.SecureBoot == nil || *vmi.Spec.Domain.Firmware.Bootloader.EFI.SecureBoot
		sev := kutil.IsSEVVMI(vmi)

		if !l.efiEnvironment.Bootable(secureBoot, sev) {
			log.Log.Errorf("EFI OVMF roms missing for booting in EFI mode with SecureBoot=%v, SEV=%v", secureBoot, sev)
			return nil, fmt.Errorf("EFI OVMF roms missing for booting in EFI mode with SecureBoot=%v, SEV=%v", secureBoot, sev)
		}

		efiConf = &converter.EFIConfiguration{
			EFICode:      l.efiEnvironment.EFICode(secureBoot, sev),
			EFIVars:      l.efiEnvironment.EFIVars(secureBoot, sev),
			SecureLoader: secureBoot,
		}
	}

	// Map the VirtualMachineInstance to the Domain
	c := &converter.ConverterContext{
		Architecture:          runtime.GOARCH,
		VirtualMachine:        vmi,
		AllowEmulation:        allowEmulation,
		CPUSet:                podCPUSet,
		IsBlockPVC:            isBlockPVCMap,
		IsBlockDV:             isBlockDVMap,
		EFIConfiguration:      efiConf,
		UseVirtioTransitional: vmi.Spec.Domain.Devices.UseVirtioTransitional != nil && *vmi.Spec.Domain.Devices.UseVirtioTransitional,
		PermanentVolumes:      permanentVolumes,
		EphemeraldiskCreator:  l.ephemeralDiskCreator,
		UseLaunchSecurity:     kutil.IsSEVVMI(vmi),
	}

	if options != nil {
		c.ExpandDisksEnabled = options.ExpandDisksEnabled
		if options.VirtualMachineSMBios != nil {
			c.SMBios = options.VirtualMachineSMBios
		}
		if options.Topology != nil {
			c.Topology = options.Topology
		}
		c.MemBalloonStatsPeriod = uint(options.MemBalloonStatsPeriod)
		// Add preallocated and thick-provisioned volumes for which we need to avoid the discard=unmap option
		c.VolumesDiscardIgnore = options.PreallocatedVolumes

		if len(options.DisksInfo) > 0 {
			l.disksInfo = options.DisksInfo
		}
	}
	c.DisksInfo = l.disksInfo

	if !isMigrationTarget {
		sriovDevices, err := sriov.CreateHostDevices(vmi)
		if err != nil {
			return nil, err
		}

		c.HotplugVolumes = hotplugVolumes
		c.SRIOVDevices = sriovDevices

		genericHostDevices, err := generic.CreateHostDevices(vmi.Spec.Domain.Devices.HostDevices)
		if err != nil {
			return nil, err
		}
		c.GenericHostDevices = genericHostDevices

		gpuHostDevices, err := gpu.CreateHostDevices(vmi.Spec.Domain.Devices.GPUs)
		if err != nil {
			return nil, err
		}
		c.GPUHostDevices = gpuHostDevices
	}

	return c, nil
}

func (l *LibvirtDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, allowEmulation bool, options *cmdv1.VirtualMachineOptions) (*api.DomainSpec, error) {
	l.domainModifyLock.Lock()
	defer l.domainModifyLock.Unlock()

	logger := log.Log.Object(vmi)

	domain := &api.Domain{}

	c, err := l.generateConverterContext(vmi, allowEmulation, options, false)
	if err != nil {
		logger.Reason(err).Error("failed to generate libvirt domain from VMI spec")
		return nil, err
	}

	if err := converter.Convert_v1_VirtualMachineInstance_To_api_Domain(vmi, domain, c); err != nil {
		logger.Error("Conversion failed.")
		return nil, err
	}

	// Set defaults which are not coming from the cluster
	api.NewDefaulter(c.Architecture).SetObjectDefaults_Domain(domain)

	dom, err := l.virConn.LookupDomainByName(domain.Spec.Name)
	if err != nil {
		// We need the domain but it does not exist, so create it
		if domainerrors.IsNotFound(err) {
			domain, err = l.preStartHook(vmi, domain, false)
			if err != nil {
				logger.Reason(err).Error("pre start setup for VirtualMachineInstance failed.")
				return nil, err
			}
			dom, err = l.setDomainSpecWithHooks(vmi, &domain.Spec)
			if err != nil {
				return nil, err
			}
			logger.Info("Domain defined.")
		} else {
			logger.Reason(err).Error(failedGetDomain)
			return nil, err
		}
	}
	defer dom.Free()
	domState, _, err := dom.GetState()
	if err != nil {
		logger.Reason(err).Error(failedGetDomainState)
		return nil, err
	}

	// TODO Suspend, Pause, ..., for now we only support reaching the running state
	// TODO for migration and error detection we also need the state change reason
	// TODO blocked state
	if cli.IsDown(domState) && !vmi.IsRunning() && !vmi.IsFinal() {
		err = l.generateCloudInitISO(vmi, &dom)
		if err != nil {
			return nil, err
		}
		createFlags := getDomainCreateFlags(vmi)
		err = dom.CreateWithFlags(createFlags)
		if err != nil {
			logger.Reason(err).
				Errorf("Failed to start VirtualMachineInstance with flags %v.", createFlags)
			return nil, err
		}
		logger.Info("Domain started.")
		if vmi.ShouldStartPaused() {
			l.paused.add(vmi.UID)
		}
	} else if cli.IsPaused(domState) && !l.paused.contains(vmi.UID) {
		// TODO: if state change reason indicates a system error, we could try something smarter
		err := dom.Resume()
		if err != nil {
			logger.Reason(err).Error("unpausing the VirtualMachineInstance failed.")
			return nil, err
		}
		logger.Info("Domain unpaused.")
	} else {
		// Nothing to do
	}

	xmlstr, err := dom.GetXMLDesc(0)
	if err != nil {
		return nil, err
	}

	var oldSpec api.DomainSpec
	err = xml.Unmarshal([]byte(xmlstr), &oldSpec)
	if err != nil {
		logger.Reason(err).Error("Parsing domain XML failed.")
		return nil, err
	}

	// Look up all the disks to detach
	for _, detachDisk := range getDetachedDisks(oldSpec.Devices.Disks, domain.Spec.Devices.Disks) {
		logger.V(1).Infof("Detaching disk %s, target %s", detachDisk.Alias.GetName(), detachDisk.Target.Device)
		detachBytes, err := xml.Marshal(detachDisk)
		if err != nil {
			logger.Reason(err).Error("marshalling detached disk failed")
			return nil, err
		}
		err = dom.DetachDevice(strings.ToLower(string(detachBytes)))
		if err != nil {
			logger.Reason(err).Error("detaching device")
			return nil, err
		}
	}
	// Look up all the disks to attach
	for _, attachDisk := range getAttachedDisks(oldSpec.Devices.Disks, domain.Spec.Devices.Disks) {
		allowAttach, err := checkIfDiskReadyToUse(getSourceFile(attachDisk))
		if err != nil {
			return nil, err
		}
		if !allowAttach {
			continue
		}
		logger.V(1).Infof("Attaching disk %s, target %s", attachDisk.Alias.GetName(), attachDisk.Target.Device)
		attachBytes, err := xml.Marshal(attachDisk)
		if err != nil {
			logger.Reason(err).Error("marshalling attached disk failed")
			return nil, err
{'fraction_non_alphanumeric': 0.11043716368962334, 'fraction_numerical': 0.004107364600248352, 'mean_word_length': 4.352790796761823, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3503264', 'n_tokens_mistral': 22615, 'n_tokens_neox': 20964, 'n_words': 5051}