<commit_before>/*
Copyright 2016-2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package libvirttools

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/jonboulle/clockwork"
	libvirtxml "github.com/libvirt/libvirt-go-xml"
	"k8s.io/apimachinery/pkg/fields"
	kubeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"

	"github.com/Mirantis/virtlet/pkg/metadata"
	"github.com/Mirantis/virtlet/pkg/utils"
	"github.com/Mirantis/virtlet/pkg/virt"
)

const (
	defaultMemory     = 1024
	defaultMemoryUnit = "MiB"
	defaultDomainType = "kvm"
	defaultEmulator   = "/usr/bin/kvm"
	noKvmDomainType   = "qemu"
	noKvmEmulator     = "/usr/bin/qemu-system-x86_64"

	domainStartCheckInterval      = 250 * time.Millisecond
	domainStartTimeout            = 10 * time.Second
	domainShutdownRetryInterval   = 5 * time.Second
	domainShutdownOnRemoveTimeout = 60 * time.Second
	domainDestroyCheckInterval    = 500 * time.Millisecond
	domainDestroyTimeout          = 5 * time.Second

	ContainerNsUuid       = "67b7fb47-7735-4b64-86d2-6d062d121966"
	defaultKubeletRootDir = "/var/lib/kubelet/pods"
	vmLogLocationPty      = "pty"
)

type domainSettings struct {
	useKvm           bool
	domainName       string
	domainUUID       string
	memory           int
	memoryUnit       string
	vcpuNum          int
	cpuShares        uint
	cpuPeriod        uint64
	cpuQuota         int64
	rootDiskFilepath string
	netNSPath        string
	cniConfig        string
	vmLogLocation    string
}

func (ds *domainSettings) createDomain() *libvirtxml.Domain {
	domainType := defaultDomainType
	emulator := defaultEmulator
	if !ds.useKvm {
		domainType = noKvmDomainType
		emulator = noKvmEmulator
	}

	scsiControllerIndex := uint(0)
	return &libvirtxml.Domain{
		Devices: &libvirtxml.DomainDeviceList{
			Emulator: "/vmwrapper",
			Inputs: []libvirtxml.DomainInput{
				{Type: "tablet", Bus: "usb"},
			},
			Graphics: []libvirtxml.DomainGraphic{
				{Type: "vnc", Port: -1},
			},
			Videos: []libvirtxml.DomainVideo{
				{Model: libvirtxml.DomainVideoModel{Type: "cirrus"}},
			},
			Controllers: []libvirtxml.DomainController{
				{Type: "scsi", Index: &scsiControllerIndex, Model: "virtio-scsi"},
			},
		},

		OS: &libvirtxml.DomainOS{
			Type: &libvirtxml.DomainOSType{Type: "hvm"},
			BootDevices: []libvirtxml.DomainBootDevice{
				{Dev: "hd"},
			},
		},

		Features: &libvirtxml.DomainFeatureList{ACPI: &libvirtxml.DomainFeature{}},

		OnPoweroff: "destroy",
		OnReboot:   "restart",
		OnCrash:    "restart",

		Type: domainType,

		Name:   ds.domainName,
		UUID:   ds.domainUUID,
		Memory: &libvirtxml.DomainMemory{Value: uint(ds.memory), Unit: ds.memoryUnit},
		VCPU:   &libvirtxml.DomainVCPU{Value: ds.vcpuNum},
		CPUTune: &libvirtxml.DomainCPUTune{
			Shares: &libvirtxml.DomainCPUTuneShares{Value: ds.cpuShares},
			Period: &libvirtxml.DomainCPUTunePeriod{Value: ds.cpuPeriod},
			Quota:  &libvirtxml.DomainCPUTuneQuota{Value: ds.cpuQuota},
		},
		MemoryBacking: &libvirtxml.DomainMemoryBacking{Locked: &libvirtxml.DomainMemoryBackingLocked{}},

		CMDLine: &libvirtxml.DomainCMDLine{
			Envs: []libvirtxml.QemuEnv{
				libvirtxml.QemuEnv{Name: "VIRTLET_EMULATOR", Value: emulator},
				libvirtxml.QemuEnv{Name: "VIRTLET_NS", Value: ds.netNSPath},
				libvirtxml.QemuEnv{Name: "VIRTLET_CNI_CONFIG", Value: ds.cniConfig},
			},
		},
	}
}

func canUseKvm() bool {
	if os.Getenv("VIRTLET_DISABLE_KVM") != "" {
		glog.V(0).Infof("VIRTLET_DISABLE_KVM env var not empty, using plain qemu")
		return false
	}
	return true
}

type VirtualizationTool struct {
	domainConn     virt.VirtDomainConnection
	volumePool     virt.VirtStoragePool
	imageManager   ImageManager
	metadataStore  metadata.MetadataStore
	clock          clockwork.Clock
	forceKVM       bool
	kubeletRootDir string
	rawDevices     []string
	volumeSource   VMVolumeSource
}

var _ VolumeOwner = &VirtualizationTool{}

func NewVirtualizationTool(domainConn virt.VirtDomainConnection, storageConn virt.VirtStorageConnection, imageManager ImageManager, metadataStore metadata.MetadataStore, volumePoolName, rawDevices string, volumeSource VMVolumeSource) (*VirtualizationTool, error) {
	volumePool, err := ensureStoragePool(storageConn, volumePoolName)
	if err != nil {
		return nil, err
	}
	return &VirtualizationTool{
		domainConn:    domainConn,
		volumePool:    volumePool,
		imageManager:  imageManager,
		metadataStore: metadataStore,
		clock:         clockwork.NewRealClock(),
		// FIXME: kubelet's --root-dir may be something other than /var/lib/kubelet
		// Need to remove it from daemonset mounts (both dev and non-dev)
		// Use 'nsenter -t 1 -m -- tar ...' or something to grab the path
		// from root namespace
		kubeletRootDir: defaultKubeletRootDir,
		rawDevices:     strings.Split(rawDevices, ","),
		volumeSource:   volumeSource,
	}, nil
}

func (v *VirtualizationTool) SetForceKVM(forceKVM bool) {
	v.forceKVM = forceKVM
}

func (v *VirtualizationTool) SetClock(clock clockwork.Clock) {
	v.clock = clock
}

func (v *VirtualizationTool) SetKubeletRootDir(kubeletRootDir string) {
	v.kubeletRootDir = kubeletRootDir
}

func (v *VirtualizationTool) getVMVolumes(config *VMConfig) ([]VMVolume, error) {
	return v.volumeSource(config, v)
}

func (v *VirtualizationTool) setupVolumes(config *VMConfig, domainDef *libvirtxml.Domain) error {
	vmVols, err := v.getVMVolumes(config)
	if err != nil {
		return err
	}
	diskDriverFactory, err := getDiskDriverFactory(config.ParsedAnnotations.DiskDriver)
	if err != nil {
		return err
	}
	volumeMap := make(map[string]string)
	var diskDrivers []diskDriver
	for n, vmVol := range vmVols {
		driver, err := diskDriverFactory(n)
		if err != nil {
			return err
		}
		diskDrivers = append(diskDrivers, driver)
		uuid := vmVol.Uuid()
		if uuid != "" {
			volumeMap[uuid] = driver.devPath()
		}
	}
	for n, vmVol := range vmVols {
		diskDef, err := vmVol.Setup(volumeMap)
		if err != nil {
			// try to tear down volumes that were already set up
			for _, vmVol := range vmVols[:n] {
				if err := vmVol.Teardown(); err != nil {
					glog.Warning("failed to tear down a volume on error: %v", err)
				}
			}
			return err
		}
		diskDef.Target = diskDrivers[n].target()
		diskDef.Address = diskDrivers[n].address()
		domainDef.Devices.Disks = append(domainDef.Devices.Disks, *diskDef)
	}
	return nil
}

func (v *VirtualizationTool) teardownVolumes(config *VMConfig) error {
	vmVols, err := v.getVMVolumes(config)
	if err != nil {
		return err
	}
	var errs []string
	for _, vmVol := range vmVols {
		if err := vmVol.Teardown(); err != nil {
			errs = append(errs, err.Error())
		}
	}
	if errs != nil {
		return fmt.Errorf("failed to tear down some of the volumes:\n%s", strings.Join(errs, "\n"))
	}
	return nil
}

func vmLogLocation() string {
	logLocation := os.Getenv("VIRTLET_VM_LOG_LOCATION")
	if logLocation == "" {
		return vmLogLocationPty
	}
	return logLocation
}

func (v *VirtualizationTool) RemoveLibvirtSandboxLog(sandboxId string) error {
	logLocation := vmLogLocation()
	if logLocation == vmLogLocationPty {
		return nil
	}
	return os.RemoveAll(filepath.Join(logLocation, sandboxId))
}

func (v *VirtualizationTool) addSerialDevicesToDomain(sandboxId, containerName string, containerAttempt uint32, domain *libvirtxml.Domain, settings domainSettings) error {
	port := uint(0)
	if settings.vmLogLocation != vmLogLocationPty {
		logDir := filepath.Join(settings.vmLogLocation, sandboxId)
		logPath := filepath.Join(logDir, fmt.Sprintf("%s_%d.log", containerName, containerAttempt))

		// Prepare directory where libvirt will store log file to.
		if _, err := os.Stat(logDir); os.IsNotExist(err) {
			if err := os.Mkdir(logDir, 0644); err != nil {
				return fmt.Errorf("failed to create vmLogDir '%s': %s", logDir, err.Error())
			}
		}

		domain.Devices.Serials = []libvirtxml.DomainSerial{
			{
				Type:   "file",
				Target: &libvirtxml.DomainSerialTarget{Port: &port},
				Source: &libvirtxml.DomainChardevSource{Path: logPath},
			},
		}
		domain.Devices.Consoles = []libvirtxml.DomainConsole{
			{
				Type:   "file",
				Target: &libvirtxml.DomainConsoleTarget{Type: "serial", Port: &port},
				Source: &libvirtxml.DomainChardevSource{Path: logPath},
			},
		}
	} else {
		domain.Devices.Serials = []libvirtxml.DomainSerial{
			{
				Type:   "pty",
				Target: &libvirtxml.DomainSerialTarget{Port: &port},
			},
		}
		domain.Devices.Consoles = []libvirtxml.DomainConsole{
			{
				Type:   "pty",
				Target: &libvirtxml.DomainConsoleTarget{Type: "serial", Port: &port},
			},
		}
	}
	return nil
}

func (v *VirtualizationTool) CreateContainer(config *VMConfig, netNSPath, cniConfig string) (string, error) {
	if err := config.LoadAnnotations(); err != nil {
		return "", err
	}

	domainUUID := utils.NewUuid5(ContainerNsUuid, config.PodSandboxId)
	// FIXME: this field should be moved to VMStatus struct (to be added)
	config.DomainUUID = domainUUID
	settings := domainSettings{
		domainUUID:    domainUUID,
		domainName:    domainUUID + "-" + config.Name,
		netNSPath:     netNSPath,
		cniConfig:     cniConfig,
		vmLogLocation: vmLogLocation(),
	}

	cloneName := "root_" + settings.domainUUID
	settings.vcpuNum = config.ParsedAnnotations.VCPUCount
	settings.memory = int(config.MemoryLimitInBytes)
	settings.cpuShares = uint(config.CpuShares)
	settings.cpuPeriod = uint64(config.CpuPeriod)
	// Specified cpu bandwidth limits for domains actually are set equal per each vCPU by libvirt
	// Thus, to limit overall VM's cpu threads consumption by set value in pod definition need to perform division
	settings.cpuQuota = config.CpuQuota / int64(settings.vcpuNum)
	settings.memoryUnit = "b"
	if settings.memory == 0 {
		settings.memory = defaultMemory
		settings.memoryUnit = defaultMemoryUnit
	}

	settings.useKvm = v.forceKVM || canUseKvm()
	domainConf := settings.createDomain()

	if err := v.setupVolumes(config, domainConf); err != nil {
		return "", err
	}

	ok := false
	defer func() {
		if ok {
			return
		}
		if err := v.removeDomain(settings.domainUUID, config); err != nil {
			glog.Warningf("Failed to remove domain %q: %v", settings.domainUUID, err)
			return
		}
		v.volumeCleanup(settings.domainUUID)
	}()

	containerAttempt := config.Attempt
	if err := v.addSerialDevicesToDomain(config.PodSandboxId, config.Name, containerAttempt, domainConf, settings); err != nil {
		return "", err
	}

	labels := map[string]string{}
	for k, v := range config.ContainerLabels {
		labels[k] = v
	}
	labels[kubetypes.KubernetesPodNameLabel] = config.PodName
	labels[kubetypes.KubernetesPodNamespaceLabel] = config.PodNamespace
	labels[kubetypes.KubernetesPodUIDLabel] = config.PodSandboxId
	labels[kubetypes.KubernetesContainerNameLabel] = config.Name

	if _, err := v.domainConn.DefineDomain(domainConf); err != nil {
		return "", err
	}

	domain, err := v.domainConn.LookupDomainByUUIDString(settings.domainUUID)
	if err == nil {
		// FIXME: do we really need this?
		// (this causes an GetInfo() call on the domain in case of libvirt)
		_, err = domain.State()
	}

	if err == nil {
		// FIXME: store VMConfig + VMStatus (to be added)
		err = v.metadataStore.SetContainer(config.Name, settings.domainUUID,
			config.PodSandboxId, config.Image, cloneName,<commit_msg>Fixed cleanup in case of failure upon CreateContainer<commit_after>/*
Copyright 2016-2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package libvirttools

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/jonboulle/clockwork"
	libvirtxml "github.com/libvirt/libvirt-go-xml"
	"k8s.io/apimachinery/pkg/fields"
	kubeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"

	"github.com/Mirantis/virtlet/pkg/metadata"
	"github.com/Mirantis/virtlet/pkg/utils"
	"github.com/Mirantis/virtlet/pkg/virt"
)

const (
	defaultMemory     = 1024
	defaultMemoryUnit = "MiB"
	defaultDomainType = "kvm"
	defaultEmulator   = "/usr/bin/kvm"
	noKvmDomainType   = "qemu"
	noKvmEmulator     = "/usr/bin/qemu-system-x86_64"

	domainStartCheckInterval      = 250 * time.Millisecond
	domainStartTimeout            = 10 * time.Second
	domainShutdownRetryInterval   = 5 * time.Second
	domainShutdownOnRemoveTimeout = 60 * time.Second
	domainDestroyCheckInterval    = 500 * time.Millisecond
	domainDestroyTimeout          = 5 * time.Second

	ContainerNsUuid       = "67b7fb47-7735-4b64-86d2-6d062d121966"
	defaultKubeletRootDir = "/var/lib/kubelet/pods"
	vmLogLocationPty      = "pty"
)

type domainSettings struct {
	useKvm           bool
	domainName       string
	domainUUID       string
	memory           int
	memoryUnit       string
	vcpuNum          int
	cpuShares        uint
	cpuPeriod        uint64
	cpuQuota         int64
	rootDiskFilepath string
	netNSPath        string
	cniConfig        string
	vmLogLocation    string
}

func (ds *domainSettings) createDomain() *libvirtxml.Domain {
	domainType := defaultDomainType
	emulator := defaultEmulator
	if !ds.useKvm {
		domainType = noKvmDomainType
		emulator = noKvmEmulator
	}

	scsiControllerIndex := uint(0)
	return &libvirtxml.Domain{
		Devices: &libvirtxml.DomainDeviceList{
			Emulator: "/vmwrapper",
			Inputs: []libvirtxml.DomainInput{
				{Type: "tablet", Bus: "usb"},
			},
			Graphics: []libvirtxml.DomainGraphic{
				{Type: "vnc", Port: -1},
			},
			Videos: []libvirtxml.DomainVideo{
				{Model: libvirtxml.DomainVideoModel{Type: "cirrus"}},
			},
			Controllers: []libvirtxml.DomainController{
				{Type: "scsi", Index: &scsiControllerIndex, Model: "virtio-scsi"},
			},
		},

		OS: &libvirtxml.DomainOS{
			Type: &libvirtxml.DomainOSType{Type: "hvm"},
			BootDevices: []libvirtxml.DomainBootDevice{
				{Dev: "hd"},
			},
		},

		Features: &libvirtxml.DomainFeatureList{ACPI: &libvirtxml.DomainFeature{}},

		OnPoweroff: "destroy",
		OnReboot:   "restart",
		OnCrash:    "restart",

		Type: domainType,

		Name:   ds.domainName,
		UUID:   ds.domainUUID,
		Memory: &libvirtxml.DomainMemory{Value: uint(ds.memory), Unit: ds.memoryUnit},
		VCPU:   &libvirtxml.DomainVCPU{Value: ds.vcpuNum},
		CPUTune: &libvirtxml.DomainCPUTune{
			Shares: &libvirtxml.DomainCPUTuneShares{Value: ds.cpuShares},
			Period: &libvirtxml.DomainCPUTunePeriod{Value: ds.cpuPeriod},
			Quota:  &libvirtxml.DomainCPUTuneQuota{Value: ds.cpuQuota},
		},
		MemoryBacking: &libvirtxml.DomainMemoryBacking{Locked: &libvirtxml.DomainMemoryBackingLocked{}},

		CMDLine: &libvirtxml.DomainCMDLine{
			Envs: []libvirtxml.QemuEnv{
				libvirtxml.QemuEnv{Name: "VIRTLET_EMULATOR", Value: emulator},
				libvirtxml.QemuEnv{Name: "VIRTLET_NS", Value: ds.netNSPath},
				libvirtxml.QemuEnv{Name: "VIRTLET_CNI_CONFIG", Value: ds.cniConfig},
			},
		},
	}
}

func canUseKvm() bool {
	if os.Getenv("VIRTLET_DISABLE_KVM") != "" {
		glog.V(0).Infof("VIRTLET_DISABLE_KVM env var not empty, using plain qemu")
		return false
	}
	return true
}

type VirtualizationTool struct {
	domainConn     virt.VirtDomainConnection
	volumePool     virt.VirtStoragePool
	imageManager   ImageManager
	metadataStore  metadata.MetadataStore
	clock          clockwork.Clock
	forceKVM       bool
	kubeletRootDir string
	rawDevices     []string
	volumeSource   VMVolumeSource
}

var _ VolumeOwner = &VirtualizationTool{}

func NewVirtualizationTool(domainConn virt.VirtDomainConnection, storageConn virt.VirtStorageConnection, imageManager ImageManager, metadataStore metadata.MetadataStore, volumePoolName, rawDevices string, volumeSource VMVolumeSource) (*VirtualizationTool, error) {
	volumePool, err := ensureStoragePool(storageConn, volumePoolName)
	if err != nil {
		return nil, err
	}
	return &VirtualizationTool{
		domainConn:    domainConn,
		volumePool:    volumePool,
		imageManager:  imageManager,
		metadataStore: metadataStore,
		clock:         clockwork.NewRealClock(),
		// FIXME: kubelet's --root-dir may be something other than /var/lib/kubelet
		// Need to remove it from daemonset mounts (both dev and non-dev)
		// Use 'nsenter -t 1 -m -- tar ...' or something to grab the path
		// from root namespace
		kubeletRootDir: defaultKubeletRootDir,
		rawDevices:     strings.Split(rawDevices, ","),
		volumeSource:   volumeSource,
	}, nil
}

func (v *VirtualizationTool) SetForceKVM(forceKVM bool) {
	v.forceKVM = forceKVM
}

func (v *VirtualizationTool) SetClock(clock clockwork.Clock) {
	v.clock = clock
}

func (v *VirtualizationTool) SetKubeletRootDir(kubeletRootDir string) {
	v.kubeletRootDir = kubeletRootDir
}

func (v *VirtualizationTool) getVMVolumes(config *VMConfig) ([]VMVolume, error) {
	return v.volumeSource(config, v)
}

func (v *VirtualizationTool) setupVolumes(config *VMConfig, domainDef *libvirtxml.Domain) error {
	vmVols, err := v.getVMVolumes(config)
	if err != nil {
		return err
	}
	diskDriverFactory, err := getDiskDriverFactory(config.ParsedAnnotations.DiskDriver)
	if err != nil {
		return err
	}
	volumeMap := make(map[string]string)
	var diskDrivers []diskDriver
	for n, vmVol := range vmVols {
		driver, err := diskDriverFactory(n)
		if err != nil {
			return err
		}
		diskDrivers = append(diskDrivers, driver)
		uuid := vmVol.Uuid()
		if uuid != "" {
			volumeMap[uuid] = driver.devPath()
		}
	}
	for n, vmVol := range vmVols {
		diskDef, err := vmVol.Setup(volumeMap)
		if err != nil {
			// try to tear down volumes that were already set up
			for _, vmVol := range vmVols[:n] {
				if err := vmVol.Teardown(); err != nil {
					glog.Warning("failed to tear down a volume on error: %v", err)
				}
			}
			return err
		}
		diskDef.Target = diskDrivers[n].target()
		diskDef.Address = diskDrivers[n].address()
		domainDef.Devices.Disks = append(domainDef.Devices.Disks, *diskDef)
	}
	return nil
}

func (v *VirtualizationTool) teardownVolumes(config *VMConfig) error {
	vmVols, err := v.getVMVolumes(config)
	if err != nil {
		return err
	}
	var errs []string
	for _, vmVol := range vmVols {
		if err := vmVol.Teardown(); err != nil {
			errs = append(errs, err.Error())
		}
	}
	if errs != nil {
		return fmt.Errorf("failed to tear down some of the volumes:\n%s", strings.Join(errs, "\n"))
	}
	return nil
}

func vmLogLocation() string {
	logLocation := os.Getenv("VIRTLET_VM_LOG_LOCATION")
	if logLocation == "" {
		return vmLogLocationPty
	}
	return logLocation
}

func (v *VirtualizationTool) RemoveLibvirtSandboxLog(sandboxId string) error {
	logLocation := vmLogLocation()
	if logLocation == vmLogLocationPty {
		return nil
	}
	return os.RemoveAll(filepath.Join(logLocation, sandboxId))
}

func (v *VirtualizationTool) addSerialDevicesToDomain(sandboxId, containerName string, containerAttempt uint32, domain *libvirtxml.Domain, settings domainSettings) error {
	port := uint(0)
	if settings.vmLogLocation != vmLogLocationPty {
		logDir := filepath.Join(settings.vmLogLocation, sandboxId)
		logPath := filepath.Join(logDir, fmt.Sprintf("%s_%d.log", containerName, containerAttempt))

		// Prepare directory where libvirt will store log file to.
		if _, err := os.Stat(logDir); os.IsNotExist(err) {
			if err := os.Mkdir(logDir, 0644); err != nil {
				return fmt.Errorf("failed to create vmLogDir '%s': %s", logDir, err.Error())
			}
		}

		domain.Devices.Serials = []libvirtxml.DomainSerial{
			{
				Type:   "file",
				Target: &libvirtxml.DomainSerialTarget{Port: &port},
				Source: &libvirtxml.DomainChardevSource{Path: logPath},
			},
		}
		domain.Devices.Consoles = []libvirtxml.DomainConsole{
			{
				Type:   "file",
				Target: &libvirtxml.DomainConsoleTarget{Type: "serial", Port: &port},
				Source: &libvirtxml.DomainChardevSource{Path: logPath},
			},
		}
	} else {
		domain.Devices.Serials = []libvirtxml.DomainSerial{
			{
				Type:   "pty",
				Target: &libvirtxml.DomainSerialTarget{Port: &port},
			},
		}
		domain.Devices.Consoles = []libvirtxml.DomainConsole{
			{
				Type:   "pty",
				Target: &libvirtxml.DomainConsoleTarget{Type: "serial", Port: &port},
			},
		}
	}
	return nil
}

func (v *VirtualizationTool) CreateContainer(config *VMConfig, netNSPath, cniConfig string) (string, error) {
	if err := config.LoadAnnotations(); err != nil {
		return "", err
	}

	domainUUID := utils.NewUuid5(ContainerNsUuid, config.PodSandboxId)
	// FIXME: this field should be moved to VMStatus struct (to be added)
	config.DomainUUID = domainUUID
	settings := domainSettings{
		domainUUID:    domainUUID,
		domainName:    domainUUID + "-" + config.Name,
		netNSPath:     netNSPath,
		cniConfig:     cniConfig,
		vmLogLocation: vmLogLocation(),
	}

	cloneName := "root_" + settings.domainUUID
	settings.vcpuNum = config.ParsedAnnotations.VCPUCount
	settings.memory = int(config.MemoryLimitInBytes)
	settings.cpuShares = uint(config.CpuShares)
	settings.cpuPeriod = uint64(config.CpuPeriod)
	// Specified cpu bandwidth limits for domains actually are set equal per each vCPU by libvirt
	// Thus, to limit overall VM's cpu threads consumption by set value in pod definition need to perform division
	settings.cpuQuota = config.CpuQuota / int64(settings.vcpuNum)
	settings.memoryUnit = "b"
	if settings.memory == 0 {
		settings.memory = defaultMemory
		settings.memoryUnit = defaultMemoryUnit
	}

	settings.useKvm = v.forceKVM || canUseKvm()
	domainConf := settings.createDomain()

	if err := v.setupVolumes(config, domainConf); err != nil {
		return "", err
	}

	ok := false
	defer func() {
		if ok {
			return
		}
		if err := v.removeDomain(settings.domainUUID, config); err != nil {
			glog.Warningf("Failed to remove domain %q: %v", settings.domainUUID, err)
			return
		}
		if err := v.teardownVolumes(config); err != nil {
			glog.Errorf("Volume teardown failed for domain %q: %v", settings.domainUUID, err)
		}
	}()

	containerAttempt := config.Attempt
	if err := v.addSerialDevicesToDomain(config.PodSandboxId, config.Name, containerAttempt, domainConf, settings); err != nil {
		return "", err
	}

	labels := map[string]string{}
	for k, v := range config.ContainerLabels {
		labels[k] = v
	}
	labels[kubetypes.KubernetesPodNameLabel] = config.PodName
	labels[kubetypes.KubernetesPodNamespaceLabel] = config.PodNamespace
	labels[kubetypes.KubernetesPodUIDLabel] = config.PodSandboxId
	labels[kubetypes.KubernetesContainerNameLabel] = config.Name

	if _, err := v.domainConn.DefineDomain(domainConf); err != nil {
		return "", err
	}

	domain, err := v.domainConn.LookupDomainByUUIDString(settings.domainUUID)
	if err == nil {
		// FIXME: do we really need this?
		// (this causes an GetInfo() call on the domain in case of libvirt)
		_, err = domain.State()
	}

	if err == nil {
		// FIXME: store VMConfig + VMStatus (to be added)
		err = v.metadataStore.SetContainer(config.Name, settings.domainUUID,
			config.PodSandboxId, config.Image, cloneName,
{'fraction_non_alphanumeric': 0.11474093483222043, 'fraction_numerical': 0.007015467838728764, 'mean_word_length': 4.131858599002386, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 80}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2251258', 'n_tokens_mistral': 8652, 'n_tokens_neox': 8032, 'n_words': 1855}