<commit_before># Copyright (C) 2010,2011  <NAME> <<EMAIL>>
# Copyright (C) 2012-2018  <NAME> <<EMAIL>>

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

"""
Main class for guest installation
"""

import base64
import errno
import hashlib
import logging
import os
import re
import shutil
import socket
import stat
import struct
import subprocess
import tempfile
import time
try:
    import urllib.parse as urlparse
except ImportError:
    import urlparse
import uuid

import M2Crypto

import guestfs

import libvirt

import lxml.etree

import oz.GuestFSManager
import oz.OzException
import oz.ozutil


class Guest(object):
    """
    Main class for guest installation.
    """
    def _discover_libvirt_type(self):
        """
        Internal method to discover the libvirt type (qemu, kvm, etc) that
        we should use, if not specified by the user.
        """
        if self.libvirt_type is None:
            doc = lxml.etree.fromstring(self.libvirt_conn.getCapabilities())

            # Libvirt calls the old intel 32-bit architecture i686, while we
            # refer to it as i386.  Do the mapping here, since we need to look
            # up the libvirt name.
            libvirtarch = self.tdl.arch
            if libvirtarch == 'i386':
                libvirtarch = 'i686'

            if len(doc.xpath("/capabilities/guest/arch[@name='%s']/domain[@type='kvm']" % (libvirtarch))) > 0:
                self.libvirt_type = 'kvm'
            elif len(doc.xpath("/capabilities/guest/arch[@name='%s']/domain[@type='qemu']" % (libvirtarch))) > 0:
                self.libvirt_type = 'qemu'
            else:
                raise oz.OzException.OzException("This host does not support virtualization type kvm or qemu for TDL arch (%s)" % (libvirtarch))

        self.log.debug("Libvirt type is %s", self.libvirt_type)

    def _discover_libvirt_bridge(self):
        """
        Internal method to discover a libvirt bridge (if necessary).
        """
        if self.bridge_name is None:
            # otherwise, try to detect a private libvirt bridge
            for netname in self.libvirt_conn.listNetworks():
                network = self.libvirt_conn.networkLookupByName(netname)

                xml = network.XMLDesc(0)
                doc = lxml.etree.fromstring(xml)

                forward = doc.xpath('/network/forward')
                if len(forward) != 1:
                    self.log.warning("Libvirt network without a forward element, skipping")
                    continue

                if forward[0].get('mode') == 'nat':
                    ips = doc.xpath('/network/ip')
                    if not ips:
                        self.log.warning("Libvirt network without an IP, skipping")
                        continue
                    for ip in ips:
                        family = ip.get("family")
                        if family is None or family == "ipv4":
                            self.bridge_name = network.bridgeName()
                            break

        if self.bridge_name is None:
            raise oz.OzException.OzException("Could not find a libvirt bridge.  Please run 'virsh net-start default' to start the default libvirt network, or see http://github.com/clalancette/oz/wiki/Oz-Network-Configuration for more information")

        self.log.debug("libvirt bridge name is %s", self.bridge_name)

    def connect_to_libvirt(self):
        """
        Method to connect to libvirt and detect various things about the
        environment.
        """
        def _libvirt_error_handler(ctxt, err):
            """
            Error callback to suppress libvirt printing to stderr by default.
            """
            pass

        libvirt.registerErrorHandler(_libvirt_error_handler, 'context')
        self.libvirt_conn = libvirt.open(self.libvirt_uri)
        self._discover_libvirt_bridge()
        self._discover_libvirt_type()

    def __init__(self, tdl, config, auto, output_disk, nicmodel, clockoffset,
                 mousetype, diskbus, iso_allowed, url_allowed, macaddress, useuefi):
        self.tdl = tdl

        # for backwards compatibility
        self.name = self.tdl.name

        if self.tdl.arch not in ["i386", "x86_64", "ppc64", "ppc64le", "aarch64", "armv7l", "s390x"]:
            raise oz.OzException.OzException("Unsupported guest arch " + self.tdl.arch)

        if os.uname()[4] in ["i386", "i586", "i686"] and self.tdl.arch == "x86_64":
            raise oz.OzException.OzException("Host machine is i386, but trying to install x86_64 guest; this cannot work")

        self.log = logging.getLogger('%s.%s' % (__name__,
                                                self.__class__.__name__))
        self.uuid = uuid.uuid4()
        self.macaddr = macaddress
        if macaddress is None:
            self.macaddr = oz.ozutil.generate_macaddress()

        # configuration from 'paths' section
        self.output_dir = oz.ozutil.config_get_path(config, 'paths',
                                                    'output_dir',
                                                    oz.ozutil.default_output_dir())

        oz.ozutil.mkdir_p(self.output_dir)

        self.data_dir = oz.ozutil.config_get_path(config, 'paths',
                                                  'data_dir',
                                                  oz.ozutil.default_data_dir())

        self.screenshot_dir = oz.ozutil.config_get_path(config, 'paths',
                                                        'screenshot_dir',
                                                        oz.ozutil.default_screenshot_dir())

        self.sshprivkey = oz.ozutil.config_get_path(config, 'paths',
                                                    'sshprivkey',
                                                    oz.ozutil.default_sshprivkey())

        # configuration from 'libvirt' section
        self.libvirt_uri = oz.ozutil.config_get_key(config, 'libvirt', 'uri',
                                                    'qemu:///system')
        self.libvirt_type = oz.ozutil.config_get_key(config, 'libvirt', 'type',
                                                     None)
        self.bridge_name = oz.ozutil.config_get_key(config, 'libvirt',
                                                    'bridge_name', None)
        self.install_cpus = oz.ozutil.config_get_key(config, 'libvirt', 'cpus',
                                                     1)
        # the memory in the configuration file is specified in megabytes, but
        # libvirt expects kilobytes, so multiply by 1024
        self.install_memory = int(oz.ozutil.config_get_key(config, 'libvirt',
                                                               'memory', 2048)) * 1024
        self.image_type = oz.ozutil.config_get_key(config, 'libvirt',
                                                   'image_type', 'raw')

        self.rng_model = oz.ozutil.config_get_key(config, 'libvirt',
                                                  'rng_model', 'random')
        self.rng_device = oz.ozutil.config_get_key(config, 'libvirt',
                                                  'rng_device', '/dev/random')
        self.rng_rate_bytes = int(oz.ozutil.config_get_key(config, 'libvirt',
                                                           'rng_rate_bytes', 1024))
        self.rng_rate_period = int(oz.ozutil.config_get_key(config, 'libvirt',
                                                           'rng_rate_period', 1000))

        # configuration from 'cache' section
        self.cache_original_media = oz.ozutil.config_get_boolean_key(config,
                                                                     'cache',
                                                                     'original_media',
                                                                     True)
        self.cache_modified_media = oz.ozutil.config_get_boolean_key(config,
                                                                     'cache',
                                                                     'modified_media',
                                                                     False)
        self.cache_jeos = oz.ozutil.config_get_boolean_key(config, 'cache',
                                                           'jeos', False)

        self.jeos_cache_dir = os.path.join(self.data_dir, "jeos")

        # configuration of "safe" ICICLE generation option
        self.safe_icicle_gen = oz.ozutil.config_get_boolean_key(config,
                                                                'icicle',
                                                                'safe_generation',
                                                                False)

        # configuration of 'timeouts' section
        self.default_install_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
                                                                    'install', 1200))
        self.inactivity_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
                                                               'inactivity', 300))
        self.boot_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
                                                         'boot', 300))
        self.shutdown_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
                                                             'shutdown', 90))

        # only pull a cached JEOS if it was built with the correct image type
        jeos_extension = self.image_type
        if self.image_type == 'raw':
            # backwards compatible
            jeos_extension = 'dsk'

        self.jeos_filename = os.path.join(self.jeos_cache_dir,
                                          self.tdl.distro + self.tdl.update + self.tdl.arch + '.' + jeos_extension)

        self.diskimage = output_disk
        if self.diskimage is None:
            ext = "." + self.image_type
            # compatibility with older versions of Oz
            if self.image_type == 'raw':
                ext = '.dsk'
            self.diskimage = os.path.join(self.output_dir, self.tdl.name + ext)

        if not os.path.isabs(self.diskimage):
            raise oz.OzException.OzException("Output disk image must be an absolute path")

        self.icicle_tmp = os.path.join(self.data_dir, "icicletmp",
                                       self.tdl.name)

        self.listen_port = oz.ozutil.get_free_port()

        self.console_listen_port = oz.ozutil.get_free_port()

        self.connect_to_libvirt()

        self.nicmodel = nicmodel
        if self.nicmodel is None:
            self.nicmodel = "rtl8139"
        self.clockoffset = clockoffset
        if self.clockoffset is None:
            self.clockoffset = "utc"
        self.mousetype = mousetype
        if self.mousetype is None:
            if self.tdl.arch in ["aarch64", "armv7l"]:
                self.mousetype = "usb"
            else:
                self.mousetype = "ps2"
        if diskbus is None or diskbus == "ide":
            self.disk_bus = "ide"
            self.disk_dev = "hda"
        elif diskbus == "virtio":
            self.disk_bus = "virtio"
            self.disk_dev = "vda"
        else:
            raise oz.OzException.OzException("Unknown diskbus type " + diskbus)

        self.rootpw = self.tdl.rootpw
        if self.rootpw is None:
            self.rootpw = "ozrootpw"

        try:
            # PPC64 directory structure differs from x86; disabling ISO boots
            if self.tdl.arch not in ["ppc64", "ppc64le"]:
                self.url = self._check_url(iso=iso_allowed, url=url_allowed)
            else:
                self.url = self._check_url(iso=False, url=url_allowed)
        except:
            self.log.debug("Install URL validation failed:", exc_info=True)
            raise

        oz.ozutil.mkdir_p(self.icicle_tmp)

        self.disksize = 10
        if self.tdl.disksize is not None:
            self.disksize = int(self.tdl.disksize)

        self.auto = auto
        if self.auto is None:
            self.auto = self.get_auto_path()

        self.log.debug("Name: %s, UUID: %s", self.tdl.name, self.uuid)
        self.log.debug("MAC: %s, distro: %s", self.macaddr, self.tdl.distro)
        self.log.debug("update: %s, arch: %s, diskimage: %s", self.tdl.update, self.tdl.arch, self.diskimage)
        self.log.debug("nicmodel: %s, clockoffset: %s", self.nicmodel, self.clockoffset)
        self.log.debug("mousetype: %s, disk_bus: %s, disk_dev: %s", self.mousetype, self.disk_bus, self.disk_dev)
        self.log.debug("icicletmp: %s, listen_port: %d", self.icicle_tmp, self.listen_port)
        self.log.debug("console_listen_port: %s", self.console_listen_port)

    def image_name(self):
        """
        Name of the image being built.
        """
        return self.name

    def output_image_path(self):
        """
        Path to the created image file.
        """
        return self.diskimage

    def get_auto_path(self):
        """
        Base method used to generate the path to the automatic installation
        file (kickstart, preseed, winnt.sif, etc).  Some subclasses override
        override this method to provide support for additional aliases.
        """
        return oz.ozutil.generate_full_auto_path(self.tdl.distro + self.tdl.update + ".auto")

    def default_auto_file(self):
        """
        Method to determine if the auto file is the default one or
        user-provided.
        """
        return self.auto == self.get_auto_path()

    def cleanup_old_guest(self):
        """
        Method to completely clean up an old guest, including deleting the
        disk file.  Use with caution!
        """
        self.log.info("Cleaning up guest named %s", self.tdl.name)
        try:
            dom = self.libvirt_conn.lookupByName(self.tdl.name)
            try:
                dom.destroy()
            except libvirt.libvirtError:
                pass
            dom.undefine()
        except libvirt.libvirtError:
            pass

        try:
            os.unlink(self.diskimage)
        except OSError as err:
            if err.errno != errno.ENOENT:
                raise

    def check_for_guest_conflict(self):
        """
        Method to check if any of our future actions will conflict with an
        already existing guest.  In particular, if a guest with the same
        name, UUID, or diskimage already exists, this throws an exception.
        """
        self.log.info("Checking for guest conflicts with %s", self.tdl.name)

        try:
            self.libvirt_conn.lookupByName(self.tdl.name)
            raise oz.OzException.OzException("Domain with name %s already exists" % (self.tdl.name))
        except libvirt.libvirtError:
            pass

        try:
            self.libvirt_conn.lookupByUUID(str(self.uuid))
            raise oz.OzException.OzException("Domain with UUID %s already exists" % (self.uuid))
        except libvirt.libvirtError:
            pass

        if os.access(self.diskimage, os.F_OK):
            raise oz.OzException.OzException("Diskimage %s already exists" % (self.diskimage))

    # the next 4 methods are intended to be overridden by the individual
    # OS backends; raise an error if they are called but not implemented

    def generate_install_media(self, force_download=False,
                               customize_or_icicle=False):
        """
        Base method for generating the install media for operating system
        installation.  This is expected to be overridden by all subclasses.
        """
        raise oz.OzException.OzException("Install media for %s%s is not implemented, install cannot continue" % (self.tdl.distro, self.tdl.update))

    def customize(self, libvirt_xml):
        """
        Base method for customizing the operating system.  This is expected
        to be overridden by subclasses that support customization.
        """
        raise oz.OzException.OzException("Customization for %s%s is not implemented" % (self.tdl.distro, self.tdl.update))

    def generate_icicle(self, libvirt_xml):
        """
        Base method for generating the ICICLE manifest from the operating
        system.  This is expect to be overridden by subclasses that support
        ICICLE generation.
        """
        raise oz.OzException.OzException("ICICLE generation for %s%s is not implemented" % (self.tdl.distro, self.tdl.update))

    # this method is intended to be an optimization if the user wants to do
    # both customize and generate_icicle
    def customize_and_generate_icicle(self, libvirt_xml):
        """
        Base method for doing operating system customization and ICICLE
        generation.  This is an optimization over doing the two steps
        separately for those classes that support customization and ICICLE
        generation.
        """
        raise oz.OzException.OzException("Customization and ICICLE generate for %s%s is not implemented" % (self.tdl.distro, self.tdl.update))

    class _InstallDev(object):
        """
        Class to hold information about an installation device.
        """
        def __init__(self, devicetype, path, bus):
            self.devicetype = devicetype
            self.path = path
            self.bus = bus

    def _generate_serial_xml(self, devices):
        """
        Method to generate the serial portion of the libvirt XML.
        """
        serial = oz.ozutil.lxml_subelement(devices, "serial", None, {'type': 'tcp'})
        oz.ozutil.lxml_subelement(serial, "source", None,
                                  {'mode': 'bind', 'host': '127.0.0.1', 'service': str(self.listen_port)})
        oz.ozutil.lxml_subelement(serial, "protocol", None, {'type': 'raw'})
        target = oz.ozutil.lxml_subelement(serial, "target", None, {'port': '1'})
        if self.tdl.arch == 's390x':
            # use a different console, as sclp can be used as most once
            target.set('type', 'sclp-serial')
            oz.ozutil.lxml_subelement(target, "model", None, {'name': 'sclplmconsole'})

    def _generate_virtio_channel(self, devices, name):
        """
        Method to generate libvirt XML for a virtio channel.
        """
        virtio = oz.ozutil.lxml_subelement(devices, "channel", None, {'type': 'tcp'})
        oz.ozutil.lxml_subelement(virtio, "source", None,
                                  {'mode': 'bind', 'host': '127.0.0.1', 'service': str(self.console_listen_port)})
        oz.ozutil.lxml_subelement(virtio, "target", None, {"type": "virtio", "name": name})

    def _generate_xml(self, bootdev, installdev, kernel=None, initrd=None,
                      cmdline=None, virtio_channel_name=None):
        """
        Method to generate libvirt XML useful for installation.
        """
        self.log.info("Generate XML for guest %s with bootdev %s", self.tdl.name, bootdev)

        # top-level domain element
        domain = lxml.etree.Element("domain", type=self.libvirt_type)
        # name element
        oz.ozutil.lxml_subelement(domain, "name", self.tdl.name)
        # memory elements
        oz.ozutil.lxml_subelement(domain, "memory", str(self.install_memory))
        oz.ozutil.lxml_subelement(domain, "currentMemory", str(self.install_memory))
        # uuid
        oz.ozutil.lxml_subelement(domain, "uuid", str(self.uuid))
        # clock offset
        oz.ozutil.lxml_subelement(domain, "clock", None, {'offset': self.clockoffset})
        # vcpu
        oz.ozutil.lxml_subelement(domain, "vcpu", str(self.install_cpus))
        # features
        features = oz.ozutil.lxml_subelement(domain, "features")
        if self.tdl.arch in ["x86_64"]:
            oz.ozutil.lxml_subelement(features, "acpi")
        if self.tdl.arch in ["aarch64"]:
            oz.ozutil.lxml_subelement(features, "acpi")
            # current edk2+linux on aarch64 puts a SPCR in place when serial is active
            # this then becomes the default tty unlike on x86 where it will continue to
            # use the uefi framebuffer/graphics console. Force linux to use both.
            cmdline = str(cmdline) + " console=ttyAMA0 console=tty0"
        if self.tdl.arch in ["x86_64"]:
            oz.ozutil.lxml_subelement(features, "apic")
            oz.ozutil.lxml_subelement(features, "pae")
        # CPU
        if self.libvirt_type == "kvm":
            # If using KVM, we always want the best CPU the host can offer
            # as we don't need to worry about live migration portability
            oz.ozutil.lxml_subelement(domain, "cpu", None, {'mode': 'host-passthrough'})
        # os
        osNode = oz.ozutil.lxml_subelement(domain, "os")
        mods = None
        if self.tdl.arch in ["aarch64", "armv7l"]:
            mods = {'arch': self.tdl.arch, 'machine': 'virt'}
        oz.ozutil.lxml_subelement(osNode, "type", "hvm", mods)
        if bootdev:
            oz.ozutil.lxml_subelement(osNode, "boot", None, {'dev': bootdev})
        if kernel:
            oz.ozutil.lxml_subelement(osNode, "kernel", kernel)
        if initrd:
            oz.ozutil.lxml_subelement(osNode, "initrd", initrd)
        if cmdline:
            oz.ozutil.lxml_subelement(osNode, "cmdline", cmdline)
        if self.tdl.arch in ["aarch64", "armv7l"]:
            loader, nvram = oz.ozutil.find_uefi_firmware(self.tdl.arch)
            oz.ozutil.lxml_subelement(osNode, "loader", loader, {'readonly': 'yes', 'type': 'pflash'})
            oz.ozutil.lxml_subelement(osNode, "nvram", None, {'template': nvram})
        # x86_64 has legacy requirements so we check for defaults as well as for edk2
        if self.tdl.arch in ["x86_64"] and self.config.useuefi == True:
            loader, nvram = oz.ozutil.find_uefi_firmware(self.tdl.arch)
            oz.ozutil.lxml_subelement(osNode, "loader", loader, {'readonly': 'yes', 'type': 'pflash'})
            oz.ozutil.lxml_subelement(osNode, "nvram", None, {'template': nvram})
        # poweroff, reboot, crash
        oz.ozutil.lxml_subelement(domain, "on_poweroff", "destroy")
        oz.ozutil.lxml_subelement(domain, "on_reboot", "destroy")
        oz.ozutil.lxml_subelement(domain, "on_crash", "destroy")
        # devices
        devices = oz.ozutil.lxml_subelement(domain, "devices")
        # graphics
        if not self.tdl.arch in ["s390x"]:
            # qemu for s390x does not support a graphical console
            oz.ozutil.lxml_subelement(devices, "graphics", None, {'port': '-1', 'type': 'vnc'})
        # network
        interface = oz.ozutil.lxml_subelement(devices, "interface", None, {'type': 'bridge'})
        oz.ozutil.lxml_subelement(interface, "source", None, {'bridge': self.bridge_name})
        oz.ozutil.lxml_subelement(interface, "mac", None, {'address': self.macaddr})
        oz.ozutil.lxml_subelement(interface, "model", None, {'type': self.nicmodel})
        # input (for non-s390x only)
        if not self.tdl.arch in ["s390x"]:
            mousedict = {'bus': self.mousetype}
            if self.mousetype == "ps2":
                mousedict['type'] = 'mouse'
            elif self.mousetype == "usb":
                mousedict['type'] = 'tablet'
            oz.ozutil.lxml_subelement(devices, "input", None, mousedict)
            if self.tdl.arch in ["aarch64", "armv7l"]:
                # Other arches add a keyboard by default, for historical reasons ARM doesn't
                # so we add it here so graphical works and hence we can get debug screenshots RHBZ 1538637
                oz.ozutil.lxml_subelement(devices, 'controller', None, {'type': 'usb', 'index': '0'})
                oz.ozutil.lxml_subelement(devices, 'input', None, {'type': 'keyboard', 'bus': 'usb'})
        # serial console pseudo TTY
        console = oz.ozutil.lxml_subelement(devices, "serial", None, {'type': 'pty'})
        oz.ozutil.lxml_subelement(console, "target", None, {'port': '0'})
        # serial
        self._generate_serial_xml(devices)
        # virtio
        if virtio_channel_name is not None:
            self._generate_virtio_channel(devices, virtio_channel_name)
            self.has_consolelog = True
        else:
            self.has_consolelog = False
        # virtio-rng
        virtioRNG = oz.ozutil.lxml_subelement(devices, "rng", None, {'model': 'virtio'})
        oz.ozutil.lxml_subelement(virtioRNG, "rate", None, {'bytes': str(self.rng_rate_bytes), 'period': str(self.rng_rate_period)})
        oz.ozutil.lxml_subelement(virtioRNG, "backend", self.rng_device, {'model': self.rng_model})
        # boot disk
        bootDisk = oz.ozutil.lxml_subelement(devices, "disk", None, {'device': 'disk', 'type': 'file'})
        oz.ozutil.lxml_subelement(bootDisk, "target", None, {'dev': self.disk_dev, 'bus': self.disk_bus})
        oz.ozutil.lxml_subelement(bootDisk, "source", None, {'file': self.diskimage})
        oz.ozutil.lxml_subelement(bootDisk, "driver", None, {'name': 'qemu', 'type': self.image_type})
        # install disk (if any)
        if not installdev:
            installdev_list = []
        elif not isinstance(installdev, list):
            installdev_list = [installdev]
        else:
            installdev_list = installdev
        for idev in installdev_list:
            install = oz.ozutil.lxml_subelement(devices, "disk", None, {'type': 'file', 'device': idev.devicetype})
            oz.ozutil.lxml_subelement(install, "source", None, {'file': idev.path})
            oz.ozutil.lxml_subelement(install, "target", None, {'dev': idev.bus})

        xml = lxml.etree.tostring(domain, pretty_print=True, encoding="unicode")
        self.log.debug("Generated XML:\n%s", xml)

        return xml

    def _internal_generate_diskimage(self, size=10, force=False,
                                     create_partition=False,
                                     image_filename=None,
                                     backing_filename=None):
        """
        Internal method to generate a diskimage.
        Set image_filename to override the default selection of self.diskimage
        Set backing_filename to force diskimage to be a writeable qcow2 snapshot
        backed by "backing_filename" which can be either a raw image or a
        qcow2 image.
        """
        if not force and os.access(self.jeos_filename, os.F_OK):
            # if we found a cached JEOS, we don't need to do anything here;
            # we'll copy the JEOS itself later on
            return

        self.log.info("Generating %dGB diskimage for %s", size, self.tdl.name)

        diskimage = self.diskimage
        if image_filename:
            diskimage = image_filename

        directory = os.path.dirname(diskimage)
        filename = os.path.basename(diskimage)

        # create the pool XML
        pool = lxml.etree.Element("pool", type="dir")
        oz.ozutil.lxml_subelement(pool, "name", "oztempdir" + str(uuid.uuid4()))
        target = oz.ozutil.lxml_subelement(pool, "target")
        oz.ozutil.lxml_subelement(target, "path", directory)
        pool_xml = lxml.etree.tostring(pool, pretty_print=True, encoding="unicode")

        # create the volume XML
        vol = lxml.etree.Element("volume", type="file")
        oz.ozutil.lxml_subelement(vol, "name", filename)
        oz.ozutil.lxml_subelement(vol, "allocation", "0")
        target = oz.ozutil.lxml_subelement(vol, "target")
        imgtype = self.image_type
        if backing_filename:
            # Only qcow2 supports image creation using a backing file
            imgtype = "qcow2"
        oz.ozutil.lxml_subelement(target, "format", None, {"type": imgtype})

        # FIXME: this makes the permissions insecure, but is needed since
        # libvirt launches guests as qemu:qemu
        permissions = oz.ozutil.lxml_subelement(target, "permissions")
        oz.ozutil.lxml_subelement(permissions, "mode", "0666")

        capacity = size
        if backing_filename:
            # FIXME: Revisit as RHBZ 958510 evolves
            # At the moment libvirt forces us to specify a size rather than
            # assuming we want to inherit the size of our backing file.
            # It may be possible to avoid this inspection step if libvirt
            # allows creation without an explicit capacity element.
            qcow_size = oz.ozutil.check_qcow_size(backing_filename)
            if qcow_size:
                capacity = qcow_size / 1024 / 1024 / 1024
                backing_format = 'qcow2'
            else:
                capacity = os.path.getsize(backing_filename) / 1024 / 1024 / 1024
                backing_format = 'raw'
            backing = oz.ozutil.lxml_subelement(vol, "backingStore")
            oz.ozutil.lxml_subelement(backing, "path", backing_filename)
            oz.ozutil.lxml_subelement(backing, "format", None,
                                      {"type": backing_format})

        oz.ozutil.lxml_subelement(vol, "capacity", str(int(capacity)), {'unit': 'G'})
        vol_xml = lxml.etree.tostring(vol, pretty_print=True, encoding="unicode")

        # sigh.  Yes, this is racy; if a pool is defined during this loop, we
        # might miss it.  I'm not quite sure how to do it better, and in any
        # case we don't expect that to happen often
        started = False
        found = False
        for poolname in self.libvirt_conn.listDefinedStoragePools() + self.libvirt_conn.listStoragePools():
            pool = self.libvirt_conn.storagePoolLookupByName(poolname)
            doc = lxml.etree.fromstring(pool.XMLDesc(0))
            res = doc.xpath('/pool/target/path')
            if len(res) != 1:
                continue
            if res[0].text == directory:
                # OK, this pool manages that directory; make sure it is running
                found = True
                if not pool.isActive():
                    pool.create(0)
                    started = True
                break

        if not found:
            pool = self.libvirt_conn.storagePoolCreateXML(pool_xml, 0)
            started = True

        def _vol_create_cb(args):
            """
            The callback used for waiting on volume creation to complete.
            """
            pool = args[0]
            vol_xml = args[1]

            try:
                pool.refresh(0)
            except libvirt.libvirtError as e:
                if e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR:
                    # libvirt returns a VIR_ERR_INTERNAL_ERROR when the
                    # refresh fails.
                    return False
                raise

            # this is a bit complicated, because of the cases that can
            # happen.  The cases are:
            #
            # 1.  The volume did not exist.  In this case, storageVolLookupByName()
            #     throws an exception, which we just ignore.  We then go on to
            #     create the volume
            # 2.  The volume did exist.  In this case, storageVolLookupByName()
            #     returns a valid volume object, and then we delete the volume
            try:
                vol = pool.storageVolLookupByName(filename)
                vol.delete(0)
            except libvirt.libvirtError as e:
                if e.get_error_code() != libvirt.VIR_ERR_NO_STORAGE_VOL:
                    raise

            pool.createXML(vol_xml, 0)

            return True

        try:
            # libvirt will not allow us to do certain operations (like a refresh)
            # while other operations are happening on a pool (like creating a new
            # volume).  Since we don't exactly know which other processes might be
            # running on the system, we retry for a while until our refresh and
            # volume creation succeed.  In most cases these operations will be fast.
            oz.ozutil.timed_loop(90, _vol_create_cb, "Waiting for volume to be created", (pool, vol_xml))
        finally:
            if started:
                pool.destroy()

        if create_partition:
            if backing_filename:
                self.log.warning("Asked to create partition against a copy-on-write snapshot - ignoring")
            else:
                g_handle = oz.GuestFSManager.GuestFS(self.diskimage, self.image_type)
                g_handle.create_msdos_partition_table()
                g_handle.cleanup()

    def generate_diskimage(self, size=10, force=False):
        """
        Method to generate a diskimage.  By default, a blank diskimage of
        10GB will be created; the caller can override this with the size
        parameter, specified in GB.  If force is False (the default), then
        a diskimage will not be created if a cached JEOS is found.  If
        force is True, a diskimage will be created regardless of whether a
        cached JEOS exists.  See the oz-install man page for more
        information about JEOS caching.
        """
        return self._internal_generate_diskimage(size, force, False)

    def _get_disks_and_interfaces(self, libvirt_xml):
        """
        Method to figure out the disks and interfaces attached to a domain.
        The method returns two lists: the first is a list of disk devices (like
        hda, hdb, etc), and the second is a list of network devices (like vnet0,
        vnet1, etc).
        """
        doc = lxml.etree.fromstring(libvirt_xml)
        disktargets = doc.xpath("/domain/devices/disk/target")

        if len(disktargets) < 1:
            raise oz.OzException.OzException("Could not find disk target")
        disks = []
        for target in disktargets:
            dev = target.get('dev')
            if dev:
                disks.append(dev)
        if not disks:
            raise oz.OzException.OzException("Could not find disk target device")
        inttargets = doc.xpath("/domain/devices/interface/target")
        if len(inttargets) < 1:
            raise oz.OzException.OzException("Could not find interface target")
        interfaces = []
        for target in inttargets:
            dev = target.get('dev')
            if dev:
                interfaces.append(dev)
        if not interfaces:
            raise oz.OzException.OzException("Could not find interface target device")

        return disks, interfaces

    def _get_disk_and_net_activity(self, libvirt_dom, disks, interfaces):
        """
        Method to collect the disk and network activity by the domain.  The
        method returns two numbers: the first is the sum of all disk activity
        from all disks, and the second is the sum of all network traffic from
        all network devices.
        """
        total_disk_req = 0
        for dev in disks:
            rd_req, rd_bytes, wr_req, wr_bytes, errs = libvirt_dom.blockStats(dev)
            total_disk_req += rd_req + wr_req

        total_net_bytes = 0
        for dev in interfaces:
            rx_bytes, rx_packets, rx_errs, rx_drop, tx_bytes, tx_packets, tx_errs, tx_drop = libvirt_dom.interfaceStats(dev)
            total_net_bytes += rx_bytes + tx_bytes

        return total_disk_req, total_net_bytes

    def _wait_for_install_finish(self, xml, max_time):
        """
        Method to wait for an installation to finish.  This will wait around
        until either the VM has gone away (at which point it is assumed the
        install was successful), or until the timeout is reached (at which
        point it is assumed the install failed and raise an exception).
        """

        libvirt_dom = self.libvirt_conn.createXML(xml, 0)

        disks, interfaces = self._get_disks_and_interfaces(libvirt_dom.XMLDesc(0))

        self.last_disk_activity = 0
        self.last_network_activity = 0
        self.inactivity_countdown = self.inactivity_timeout
        self.saved_exception = None
        if self.has_consolelog:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.settimeout(1)
            self.sock.connect(('127.0.0.1', self.console_listen_port))

        def _finish_cb(self):
            '''
            A callback for _looper to deal with waiting for a guest to finish installing.
            '''
            if self.inactivity_countdown <= 0:
                return True
            try:
                total_disk_req, total_net_bytes = self._get_disk_and_net_activity(libvirt_dom, disks, interfaces)
            except libvirt.libvirtError as e:
                # we save the exception here because we want to raise it later
                # if this was a "real" exception
                self.saved_exception = e
                return True

            # rd_req and wr_req are the *total* number of disk read requests and
            # write requests ever made for this domain.  Similarly rd_bytes and
            # wr_bytes are the total number of network bytes read or written
            # for this domain

            # we define activity as having done a read or write request on the
            # install disk, or having done at least 4KB of network transfers in
            # the last second.  The thinking is that if the installer is putting
            # bits on disk, there will be disk activity, so we should keep
            # waiting.  On the other hand, the installer might be downloading
            # bits to eventually install on disk, so we look for network
            # activity as well.  We say that transfers of at least 4KB must be
            # made, however, to try to reduce false positives from things like
            # ARP requests

            if (total_disk_req == self.last_disk_activity) and (total_net_bytes < (self.last_network_activity + 4096)):
                # if we saw no read or write requests since the last iteration,
                # decrement our activity timer
                self.inactivity_countdown -= 1
            else:
                # if we did see some activity, then we can reset the timer
                self.inactivity_countdown = self.inactivity_timeout

            self.last_disk_activity = total_disk_req
            self.last_network_activity = total_net_bytes

            if self.has_consolelog:
                try:
                    # note that we have to build the data up here, since there
                    # is no guarantee that we will get the whole write in one go
                    data = self.sock.recv(65536)
                    if data:
                        self.log.debug(data.decode('utf-8'))<commit_msg>Don't abort when the data in _wait_for_install_finish isn't valid utf-8<commit_after># Copyright (C) 2010,2011  <NAME> <<EMAIL>>
# Copyright (C) 2012-2018  <NAME> <<EMAIL>>

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

"""
Main class for guest installation
"""

import base64
import errno
import hashlib
import logging
import os
import re
import shutil
import socket
import stat
import struct
import subprocess
import tempfile
import time
try:
    import urllib.parse as urlparse
except ImportError:
    import urlparse
import uuid

import M2Crypto

import guestfs

import libvirt

import lxml.etree

import oz.GuestFSManager
import oz.OzException
import oz.ozutil


class Guest(object):
    """
    Main class for guest installation.
    """
    def _discover_libvirt_type(self):
        """
        Internal method to discover the libvirt type (qemu, kvm, etc) that
        we should use, if not specified by the user.
        """
        if self.libvirt_type is None:
            doc = lxml.etree.fromstring(self.libvirt_conn.getCapabilities())

            # Libvirt calls the old intel 32-bit architecture i686, while we
            # refer to it as i386.  Do the mapping here, since we need to look
            # up the libvirt name.
            libvirtarch = self.tdl.arch
            if libvirtarch == 'i386':
                libvirtarch = 'i686'

            if len(doc.xpath("/capabilities/guest/arch[@name='%s']/domain[@type='kvm']" % (libvirtarch))) > 0:
                self.libvirt_type = 'kvm'
            elif len(doc.xpath("/capabilities/guest/arch[@name='%s']/domain[@type='qemu']" % (libvirtarch))) > 0:
                self.libvirt_type = 'qemu'
            else:
                raise oz.OzException.OzException("This host does not support virtualization type kvm or qemu for TDL arch (%s)" % (libvirtarch))

        self.log.debug("Libvirt type is %s", self.libvirt_type)

    def _discover_libvirt_bridge(self):
        """
        Internal method to discover a libvirt bridge (if necessary).
        """
        if self.bridge_name is None:
            # otherwise, try to detect a private libvirt bridge
            for netname in self.libvirt_conn.listNetworks():
                network = self.libvirt_conn.networkLookupByName(netname)

                xml = network.XMLDesc(0)
                doc = lxml.etree.fromstring(xml)

                forward = doc.xpath('/network/forward')
                if len(forward) != 1:
                    self.log.warning("Libvirt network without a forward element, skipping")
                    continue

                if forward[0].get('mode') == 'nat':
                    ips = doc.xpath('/network/ip')
                    if not ips:
                        self.log.warning("Libvirt network without an IP, skipping")
                        continue
                    for ip in ips:
                        family = ip.get("family")
                        if family is None or family == "ipv4":
                            self.bridge_name = network.bridgeName()
                            break

        if self.bridge_name is None:
            raise oz.OzException.OzException("Could not find a libvirt bridge.  Please run 'virsh net-start default' to start the default libvirt network, or see http://github.com/clalancette/oz/wiki/Oz-Network-Configuration for more information")

        self.log.debug("libvirt bridge name is %s", self.bridge_name)

    def connect_to_libvirt(self):
        """
        Method to connect to libvirt and detect various things about the
        environment.
        """
        def _libvirt_error_handler(ctxt, err):
            """
            Error callback to suppress libvirt printing to stderr by default.
            """
            pass

        libvirt.registerErrorHandler(_libvirt_error_handler, 'context')
        self.libvirt_conn = libvirt.open(self.libvirt_uri)
        self._discover_libvirt_bridge()
        self._discover_libvirt_type()

    def __init__(self, tdl, config, auto, output_disk, nicmodel, clockoffset,
                 mousetype, diskbus, iso_allowed, url_allowed, macaddress, useuefi):
        self.tdl = tdl

        # for backwards compatibility
        self.name = self.tdl.name

        if self.tdl.arch not in ["i386", "x86_64", "ppc64", "ppc64le", "aarch64", "armv7l", "s390x"]:
            raise oz.OzException.OzException("Unsupported guest arch " + self.tdl.arch)

        if os.uname()[4] in ["i386", "i586", "i686"] and self.tdl.arch == "x86_64":
            raise oz.OzException.OzException("Host machine is i386, but trying to install x86_64 guest; this cannot work")

        self.log = logging.getLogger('%s.%s' % (__name__,
                                                self.__class__.__name__))
        self.uuid = uuid.uuid4()
        self.macaddr = macaddress
        if macaddress is None:
            self.macaddr = oz.ozutil.generate_macaddress()

        # configuration from 'paths' section
        self.output_dir = oz.ozutil.config_get_path(config, 'paths',
                                                    'output_dir',
                                                    oz.ozutil.default_output_dir())

        oz.ozutil.mkdir_p(self.output_dir)

        self.data_dir = oz.ozutil.config_get_path(config, 'paths',
                                                  'data_dir',
                                                  oz.ozutil.default_data_dir())

        self.screenshot_dir = oz.ozutil.config_get_path(config, 'paths',
                                                        'screenshot_dir',
                                                        oz.ozutil.default_screenshot_dir())

        self.sshprivkey = oz.ozutil.config_get_path(config, 'paths',
                                                    'sshprivkey',
                                                    oz.ozutil.default_sshprivkey())

        # configuration from 'libvirt' section
        self.libvirt_uri = oz.ozutil.config_get_key(config, 'libvirt', 'uri',
                                                    'qemu:///system')
        self.libvirt_type = oz.ozutil.config_get_key(config, 'libvirt', 'type',
                                                     None)
        self.bridge_name = oz.ozutil.config_get_key(config, 'libvirt',
                                                    'bridge_name', None)
        self.install_cpus = oz.ozutil.config_get_key(config, 'libvirt', 'cpus',
                                                     1)
        # the memory in the configuration file is specified in megabytes, but
        # libvirt expects kilobytes, so multiply by 1024
        self.install_memory = int(oz.ozutil.config_get_key(config, 'libvirt',
                                                               'memory', 2048)) * 1024
        self.image_type = oz.ozutil.config_get_key(config, 'libvirt',
                                                   'image_type', 'raw')

        self.rng_model = oz.ozutil.config_get_key(config, 'libvirt',
                                                  'rng_model', 'random')
        self.rng_device = oz.ozutil.config_get_key(config, 'libvirt',
                                                  'rng_device', '/dev/random')
        self.rng_rate_bytes = int(oz.ozutil.config_get_key(config, 'libvirt',
                                                           'rng_rate_bytes', 1024))
        self.rng_rate_period = int(oz.ozutil.config_get_key(config, 'libvirt',
                                                           'rng_rate_period', 1000))

        # configuration from 'cache' section
        self.cache_original_media = oz.ozutil.config_get_boolean_key(config,
                                                                     'cache',
                                                                     'original_media',
                                                                     True)
        self.cache_modified_media = oz.ozutil.config_get_boolean_key(config,
                                                                     'cache',
                                                                     'modified_media',
                                                                     False)
        self.cache_jeos = oz.ozutil.config_get_boolean_key(config, 'cache',
                                                           'jeos', False)

        self.jeos_cache_dir = os.path.join(self.data_dir, "jeos")

        # configuration of "safe" ICICLE generation option
        self.safe_icicle_gen = oz.ozutil.config_get_boolean_key(config,
                                                                'icicle',
                                                                'safe_generation',
                                                                False)

        # configuration of 'timeouts' section
        self.default_install_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
                                                                    'install', 1200))
        self.inactivity_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
                                                               'inactivity', 300))
        self.boot_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
                                                         'boot', 300))
        self.shutdown_timeout = int(oz.ozutil.config_get_key(config, 'timeouts',
                                                             'shutdown', 90))

        # only pull a cached JEOS if it was built with the correct image type
        jeos_extension = self.image_type
        if self.image_type == 'raw':
            # backwards compatible
            jeos_extension = 'dsk'

        self.jeos_filename = os.path.join(self.jeos_cache_dir,
                                          self.tdl.distro + self.tdl.update + self.tdl.arch + '.' + jeos_extension)

        self.diskimage = output_disk
        if self.diskimage is None:
            ext = "." + self.image_type
            # compatibility with older versions of Oz
            if self.image_type == 'raw':
                ext = '.dsk'
            self.diskimage = os.path.join(self.output_dir, self.tdl.name + ext)

        if not os.path.isabs(self.diskimage):
            raise oz.OzException.OzException("Output disk image must be an absolute path")

        self.icicle_tmp = os.path.join(self.data_dir, "icicletmp",
                                       self.tdl.name)

        self.listen_port = oz.ozutil.get_free_port()

        self.console_listen_port = oz.ozutil.get_free_port()

        self.connect_to_libvirt()

        self.nicmodel = nicmodel
        if self.nicmodel is None:
            self.nicmodel = "rtl8139"
        self.clockoffset = clockoffset
        if self.clockoffset is None:
            self.clockoffset = "utc"
        self.mousetype = mousetype
        if self.mousetype is None:
            if self.tdl.arch in ["aarch64", "armv7l"]:
                self.mousetype = "usb"
            else:
                self.mousetype = "ps2"
        if diskbus is None or diskbus == "ide":
            self.disk_bus = "ide"
            self.disk_dev = "hda"
        elif diskbus == "virtio":
            self.disk_bus = "virtio"
            self.disk_dev = "vda"
        else:
            raise oz.OzException.OzException("Unknown diskbus type " + diskbus)

        self.rootpw = self.tdl.rootpw
        if self.rootpw is None:
            self.rootpw = "ozrootpw"

        try:
            # PPC64 directory structure differs from x86; disabling ISO boots
            if self.tdl.arch not in ["ppc64", "ppc64le"]:
                self.url = self._check_url(iso=iso_allowed, url=url_allowed)
            else:
                self.url = self._check_url(iso=False, url=url_allowed)
        except:
            self.log.debug("Install URL validation failed:", exc_info=True)
            raise

        oz.ozutil.mkdir_p(self.icicle_tmp)

        self.disksize = 10
        if self.tdl.disksize is not None:
            self.disksize = int(self.tdl.disksize)

        self.auto = auto
        if self.auto is None:
            self.auto = self.get_auto_path()

        self.log.debug("Name: %s, UUID: %s", self.tdl.name, self.uuid)
        self.log.debug("MAC: %s, distro: %s", self.macaddr, self.tdl.distro)
        self.log.debug("update: %s, arch: %s, diskimage: %s", self.tdl.update, self.tdl.arch, self.diskimage)
        self.log.debug("nicmodel: %s, clockoffset: %s", self.nicmodel, self.clockoffset)
        self.log.debug("mousetype: %s, disk_bus: %s, disk_dev: %s", self.mousetype, self.disk_bus, self.disk_dev)
        self.log.debug("icicletmp: %s, listen_port: %d", self.icicle_tmp, self.listen_port)
        self.log.debug("console_listen_port: %s", self.console_listen_port)

    def image_name(self):
        """
        Name of the image being built.
        """
        return self.name

    def output_image_path(self):
        """
        Path to the created image file.
        """
        return self.diskimage

    def get_auto_path(self):
        """
        Base method used to generate the path to the automatic installation
        file (kickstart, preseed, winnt.sif, etc).  Some subclasses override
        override this method to provide support for additional aliases.
        """
        return oz.ozutil.generate_full_auto_path(self.tdl.distro + self.tdl.update + ".auto")

    def default_auto_file(self):
        """
        Method to determine if the auto file is the default one or
        user-provided.
        """
        return self.auto == self.get_auto_path()

    def cleanup_old_guest(self):
        """
        Method to completely clean up an old guest, including deleting the
        disk file.  Use with caution!
        """
        self.log.info("Cleaning up guest named %s", self.tdl.name)
        try:
            dom = self.libvirt_conn.lookupByName(self.tdl.name)
            try:
                dom.destroy()
            except libvirt.libvirtError:
                pass
            dom.undefine()
        except libvirt.libvirtError:
            pass

        try:
            os.unlink(self.diskimage)
        except OSError as err:
            if err.errno != errno.ENOENT:
                raise

    def check_for_guest_conflict(self):
        """
        Method to check if any of our future actions will conflict with an
        already existing guest.  In particular, if a guest with the same
        name, UUID, or diskimage already exists, this throws an exception.
        """
        self.log.info("Checking for guest conflicts with %s", self.tdl.name)

        try:
            self.libvirt_conn.lookupByName(self.tdl.name)
            raise oz.OzException.OzException("Domain with name %s already exists" % (self.tdl.name))
        except libvirt.libvirtError:
            pass

        try:
            self.libvirt_conn.lookupByUUID(str(self.uuid))
            raise oz.OzException.OzException("Domain with UUID %s already exists" % (self.uuid))
        except libvirt.libvirtError:
            pass

        if os.access(self.diskimage, os.F_OK):
            raise oz.OzException.OzException("Diskimage %s already exists" % (self.diskimage))

    # the next 4 methods are intended to be overridden by the individual
    # OS backends; raise an error if they are called but not implemented

    def generate_install_media(self, force_download=False,
                               customize_or_icicle=False):
        """
        Base method for generating the install media for operating system
        installation.  This is expected to be overridden by all subclasses.
        """
        raise oz.OzException.OzException("Install media for %s%s is not implemented, install cannot continue" % (self.tdl.distro, self.tdl.update))

    def customize(self, libvirt_xml):
        """
        Base method for customizing the operating system.  This is expected
        to be overridden by subclasses that support customization.
        """
        raise oz.OzException.OzException("Customization for %s%s is not implemented" % (self.tdl.distro, self.tdl.update))

    def generate_icicle(self, libvirt_xml):
        """
        Base method for generating the ICICLE manifest from the operating
        system.  This is expect to be overridden by subclasses that support
        ICICLE generation.
        """
        raise oz.OzException.OzException("ICICLE generation for %s%s is not implemented" % (self.tdl.distro, self.tdl.update))

    # this method is intended to be an optimization if the user wants to do
    # both customize and generate_icicle
    def customize_and_generate_icicle(self, libvirt_xml):
        """
        Base method for doing operating system customization and ICICLE
        generation.  This is an optimization over doing the two steps
        separately for those classes that support customization and ICICLE
        generation.
        """
        raise oz.OzException.OzException("Customization and ICICLE generate for %s%s is not implemented" % (self.tdl.distro, self.tdl.update))

    class _InstallDev(object):
        """
        Class to hold information about an installation device.
        """
        def __init__(self, devicetype, path, bus):
            self.devicetype = devicetype
            self.path = path
            self.bus = bus

    def _generate_serial_xml(self, devices):
        """
        Method to generate the serial portion of the libvirt XML.
        """
        serial = oz.ozutil.lxml_subelement(devices, "serial", None, {'type': 'tcp'})
        oz.ozutil.lxml_subelement(serial, "source", None,
                                  {'mode': 'bind', 'host': '127.0.0.1', 'service': str(self.listen_port)})
        oz.ozutil.lxml_subelement(serial, "protocol", None, {'type': 'raw'})
        target = oz.ozutil.lxml_subelement(serial, "target", None, {'port': '1'})
        if self.tdl.arch == 's390x':
            # use a different console, as sclp can be used as most once
            target.set('type', 'sclp-serial')
            oz.ozutil.lxml_subelement(target, "model", None, {'name': 'sclplmconsole'})

    def _generate_virtio_channel(self, devices, name):
        """
        Method to generate libvirt XML for a virtio channel.
        """
        virtio = oz.ozutil.lxml_subelement(devices, "channel", None, {'type': 'tcp'})
        oz.ozutil.lxml_subelement(virtio, "source", None,
                                  {'mode': 'bind', 'host': '127.0.0.1', 'service': str(self.console_listen_port)})
        oz.ozutil.lxml_subelement(virtio, "target", None, {"type": "virtio", "name": name})

    def _generate_xml(self, bootdev, installdev, kernel=None, initrd=None,
                      cmdline=None, virtio_channel_name=None):
        """
        Method to generate libvirt XML useful for installation.
        """
        self.log.info("Generate XML for guest %s with bootdev %s", self.tdl.name, bootdev)

        # top-level domain element
        domain = lxml.etree.Element("domain", type=self.libvirt_type)
        # name element
        oz.ozutil.lxml_subelement(domain, "name", self.tdl.name)
        # memory elements
        oz.ozutil.lxml_subelement(domain, "memory", str(self.install_memory))
        oz.ozutil.lxml_subelement(domain, "currentMemory", str(self.install_memory))
        # uuid
        oz.ozutil.lxml_subelement(domain, "uuid", str(self.uuid))
        # clock offset
        oz.ozutil.lxml_subelement(domain, "clock", None, {'offset': self.clockoffset})
        # vcpu
        oz.ozutil.lxml_subelement(domain, "vcpu", str(self.install_cpus))
        # features
        features = oz.ozutil.lxml_subelement(domain, "features")
        if self.tdl.arch in ["x86_64"]:
            oz.ozutil.lxml_subelement(features, "acpi")
        if self.tdl.arch in ["aarch64"]:
            oz.ozutil.lxml_subelement(features, "acpi")
            # current edk2+linux on aarch64 puts a SPCR in place when serial is active
            # this then becomes the default tty unlike on x86 where it will continue to
            # use the uefi framebuffer/graphics console. Force linux to use both.
            cmdline = str(cmdline) + " console=ttyAMA0 console=tty0"
        if self.tdl.arch in ["x86_64"]:
            oz.ozutil.lxml_subelement(features, "apic")
            oz.ozutil.lxml_subelement(features, "pae")
        # CPU
        if self.libvirt_type == "kvm":
            # If using KVM, we always want the best CPU the host can offer
            # as we don't need to worry about live migration portability
            oz.ozutil.lxml_subelement(domain, "cpu", None, {'mode': 'host-passthrough'})
        # os
        osNode = oz.ozutil.lxml_subelement(domain, "os")
        mods = None
        if self.tdl.arch in ["aarch64", "armv7l"]:
            mods = {'arch': self.tdl.arch, 'machine': 'virt'}
        oz.ozutil.lxml_subelement(osNode, "type", "hvm", mods)
        if bootdev:
            oz.ozutil.lxml_subelement(osNode, "boot", None, {'dev': bootdev})
        if kernel:
            oz.ozutil.lxml_subelement(osNode, "kernel", kernel)
        if initrd:
            oz.ozutil.lxml_subelement(osNode, "initrd", initrd)
        if cmdline:
            oz.ozutil.lxml_subelement(osNode, "cmdline", cmdline)
        if self.tdl.arch in ["aarch64", "armv7l"]:
            loader, nvram = oz.ozutil.find_uefi_firmware(self.tdl.arch)
            oz.ozutil.lxml_subelement(osNode, "loader", loader, {'readonly': 'yes', 'type': 'pflash'})
            oz.ozutil.lxml_subelement(osNode, "nvram", None, {'template': nvram})
        # x86_64 has legacy requirements so we check for defaults as well as for edk2
        if self.tdl.arch in ["x86_64"] and self.config.useuefi == True:
            loader, nvram = oz.ozutil.find_uefi_firmware(self.tdl.arch)
            oz.ozutil.lxml_subelement(osNode, "loader", loader, {'readonly': 'yes', 'type': 'pflash'})
            oz.ozutil.lxml_subelement(osNode, "nvram", None, {'template': nvram})
        # poweroff, reboot, crash
        oz.ozutil.lxml_subelement(domain, "on_poweroff", "destroy")
        oz.ozutil.lxml_subelement(domain, "on_reboot", "destroy")
        oz.ozutil.lxml_subelement(domain, "on_crash", "destroy")
        # devices
        devices = oz.ozutil.lxml_subelement(domain, "devices")
        # graphics
        if not self.tdl.arch in ["s390x"]:
            # qemu for s390x does not support a graphical console
            oz.ozutil.lxml_subelement(devices, "graphics", None, {'port': '-1', 'type': 'vnc'})
        # network
        interface = oz.ozutil.lxml_subelement(devices, "interface", None, {'type': 'bridge'})
        oz.ozutil.lxml_subelement(interface, "source", None, {'bridge': self.bridge_name})
        oz.ozutil.lxml_subelement(interface, "mac", None, {'address': self.macaddr})
        oz.ozutil.lxml_subelement(interface, "model", None, {'type': self.nicmodel})
        # input (for non-s390x only)
        if not self.tdl.arch in ["s390x"]:
            mousedict = {'bus': self.mousetype}
            if self.mousetype == "ps2":
                mousedict['type'] = 'mouse'
            elif self.mousetype == "usb":
                mousedict['type'] = 'tablet'
            oz.ozutil.lxml_subelement(devices, "input", None, mousedict)
            if self.tdl.arch in ["aarch64", "armv7l"]:
                # Other arches add a keyboard by default, for historical reasons ARM doesn't
                # so we add it here so graphical works and hence we can get debug screenshots RHBZ 1538637
                oz.ozutil.lxml_subelement(devices, 'controller', None, {'type': 'usb', 'index': '0'})
                oz.ozutil.lxml_subelement(devices, 'input', None, {'type': 'keyboard', 'bus': 'usb'})
        # serial console pseudo TTY
        console = oz.ozutil.lxml_subelement(devices, "serial", None, {'type': 'pty'})
        oz.ozutil.lxml_subelement(console, "target", None, {'port': '0'})
        # serial
        self._generate_serial_xml(devices)
        # virtio
        if virtio_channel_name is not None:
            self._generate_virtio_channel(devices, virtio_channel_name)
            self.has_consolelog = True
        else:
            self.has_consolelog = False
        # virtio-rng
        virtioRNG = oz.ozutil.lxml_subelement(devices, "rng", None, {'model': 'virtio'})
        oz.ozutil.lxml_subelement(virtioRNG, "rate", None, {'bytes': str(self.rng_rate_bytes), 'period': str(self.rng_rate_period)})
        oz.ozutil.lxml_subelement(virtioRNG, "backend", self.rng_device, {'model': self.rng_model})
        # boot disk
        bootDisk = oz.ozutil.lxml_subelement(devices, "disk", None, {'device': 'disk', 'type': 'file'})
        oz.ozutil.lxml_subelement(bootDisk, "target", None, {'dev': self.disk_dev, 'bus': self.disk_bus})
        oz.ozutil.lxml_subelement(bootDisk, "source", None, {'file': self.diskimage})
        oz.ozutil.lxml_subelement(bootDisk, "driver", None, {'name': 'qemu', 'type': self.image_type})
        # install disk (if any)
        if not installdev:
            installdev_list = []
        elif not isinstance(installdev, list):
            installdev_list = [installdev]
        else:
            installdev_list = installdev
        for idev in installdev_list:
            install = oz.ozutil.lxml_subelement(devices, "disk", None, {'type': 'file', 'device': idev.devicetype})
            oz.ozutil.lxml_subelement(install, "source", None, {'file': idev.path})
            oz.ozutil.lxml_subelement(install, "target", None, {'dev': idev.bus})

        xml = lxml.etree.tostring(domain, pretty_print=True, encoding="unicode")
        self.log.debug("Generated XML:\n%s", xml)

        return xml

    def _internal_generate_diskimage(self, size=10, force=False,
                                     create_partition=False,
                                     image_filename=None,
                                     backing_filename=None):
        """
        Internal method to generate a diskimage.
        Set image_filename to override the default selection of self.diskimage
        Set backing_filename to force diskimage to be a writeable qcow2 snapshot
        backed by "backing_filename" which can be either a raw image or a
        qcow2 image.
        """
        if not force and os.access(self.jeos_filename, os.F_OK):
            # if we found a cached JEOS, we don't need to do anything here;
            # we'll copy the JEOS itself later on
            return

        self.log.info("Generating %dGB diskimage for %s", size, self.tdl.name)

        diskimage = self.diskimage
        if image_filename:
            diskimage = image_filename

        directory = os.path.dirname(diskimage)
        filename = os.path.basename(diskimage)

        # create the pool XML
        pool = lxml.etree.Element("pool", type="dir")
        oz.ozutil.lxml_subelement(pool, "name", "oztempdir" + str(uuid.uuid4()))
        target = oz.ozutil.lxml_subelement(pool, "target")
        oz.ozutil.lxml_subelement(target, "path", directory)
        pool_xml = lxml.etree.tostring(pool, pretty_print=True, encoding="unicode")

        # create the volume XML
        vol = lxml.etree.Element("volume", type="file")
        oz.ozutil.lxml_subelement(vol, "name", filename)
        oz.ozutil.lxml_subelement(vol, "allocation", "0")
        target = oz.ozutil.lxml_subelement(vol, "target")
        imgtype = self.image_type
        if backing_filename:
            # Only qcow2 supports image creation using a backing file
            imgtype = "qcow2"
        oz.ozutil.lxml_subelement(target, "format", None, {"type": imgtype})

        # FIXME: this makes the permissions insecure, but is needed since
        # libvirt launches guests as qemu:qemu
        permissions = oz.ozutil.lxml_subelement(target, "permissions")
        oz.ozutil.lxml_subelement(permissions, "mode", "0666")

        capacity = size
        if backing_filename:
            # FIXME: Revisit as RHBZ 958510 evolves
            # At the moment libvirt forces us to specify a size rather than
            # assuming we want to inherit the size of our backing file.
            # It may be possible to avoid this inspection step if libvirt
            # allows creation without an explicit capacity element.
            qcow_size = oz.ozutil.check_qcow_size(backing_filename)
            if qcow_size:
                capacity = qcow_size / 1024 / 1024 / 1024
                backing_format = 'qcow2'
            else:
                capacity = os.path.getsize(backing_filename) / 1024 / 1024 / 1024
                backing_format = 'raw'
            backing = oz.ozutil.lxml_subelement(vol, "backingStore")
            oz.ozutil.lxml_subelement(backing, "path", backing_filename)
            oz.ozutil.lxml_subelement(backing, "format", None,
                                      {"type": backing_format})

        oz.ozutil.lxml_subelement(vol, "capacity", str(int(capacity)), {'unit': 'G'})
        vol_xml = lxml.etree.tostring(vol, pretty_print=True, encoding="unicode")

        # sigh.  Yes, this is racy; if a pool is defined during this loop, we
        # might miss it.  I'm not quite sure how to do it better, and in any
        # case we don't expect that to happen often
        started = False
        found = False
        for poolname in self.libvirt_conn.listDefinedStoragePools() + self.libvirt_conn.listStoragePools():
            pool = self.libvirt_conn.storagePoolLookupByName(poolname)
            doc = lxml.etree.fromstring(pool.XMLDesc(0))
            res = doc.xpath('/pool/target/path')
            if len(res) != 1:
                continue
            if res[0].text == directory:
                # OK, this pool manages that directory; make sure it is running
                found = True
                if not pool.isActive():
                    pool.create(0)
                    started = True
                break

        if not found:
            pool = self.libvirt_conn.storagePoolCreateXML(pool_xml, 0)
            started = True

        def _vol_create_cb(args):
            """
            The callback used for waiting on volume creation to complete.
            """
            pool = args[0]
            vol_xml = args[1]

            try:
                pool.refresh(0)
            except libvirt.libvirtError as e:
                if e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR:
                    # libvirt returns a VIR_ERR_INTERNAL_ERROR when the
                    # refresh fails.
                    return False
                raise

            # this is a bit complicated, because of the cases that can
            # happen.  The cases are:
            #
            # 1.  The volume did not exist.  In this case, storageVolLookupByName()
            #     throws an exception, which we just ignore.  We then go on to
            #     create the volume
            # 2.  The volume did exist.  In this case, storageVolLookupByName()
            #     returns a valid volume object, and then we delete the volume
            try:
                vol = pool.storageVolLookupByName(filename)
                vol.delete(0)
            except libvirt.libvirtError as e:
                if e.get_error_code() != libvirt.VIR_ERR_NO_STORAGE_VOL:
                    raise

            pool.createXML(vol_xml, 0)

            return True

        try:
            # libvirt will not allow us to do certain operations (like a refresh)
            # while other operations are happening on a pool (like creating a new
            # volume).  Since we don't exactly know which other processes might be
            # running on the system, we retry for a while until our refresh and
            # volume creation succeed.  In most cases these operations will be fast.
            oz.ozutil.timed_loop(90, _vol_create_cb, "Waiting for volume to be created", (pool, vol_xml))
        finally:
            if started:
                pool.destroy()

        if create_partition:
            if backing_filename:
                self.log.warning("Asked to create partition against a copy-on-write snapshot - ignoring")
            else:
                g_handle = oz.GuestFSManager.GuestFS(self.diskimage, self.image_type)
                g_handle.create_msdos_partition_table()
                g_handle.cleanup()

    def generate_diskimage(self, size=10, force=False):
        """
        Method to generate a diskimage.  By default, a blank diskimage of
        10GB will be created; the caller can override this with the size
        parameter, specified in GB.  If force is False (the default), then
        a diskimage will not be created if a cached JEOS is found.  If
        force is True, a diskimage will be created regardless of whether a
        cached JEOS exists.  See the oz-install man page for more
        information about JEOS caching.
        """
        return self._internal_generate_diskimage(size, force, False)

    def _get_disks_and_interfaces(self, libvirt_xml):
        """
        Method to figure out the disks and interfaces attached to a domain.
        The method returns two lists: the first is a list of disk devices (like
        hda, hdb, etc), and the second is a list of network devices (like vnet0,
        vnet1, etc).
        """
        doc = lxml.etree.fromstring(libvirt_xml)
        disktargets = doc.xpath("/domain/devices/disk/target")

        if len(disktargets) < 1:
            raise oz.OzException.OzException("Could not find disk target")
        disks = []
        for target in disktargets:
            dev = target.get('dev')
            if dev:
                disks.append(dev)
        if not disks:
            raise oz.OzException.OzException("Could not find disk target device")
        inttargets = doc.xpath("/domain/devices/interface/target")
        if len(inttargets) < 1:
            raise oz.OzException.OzException("Could not find interface target")
        interfaces = []
        for target in inttargets:
            dev = target.get('dev')
            if dev:
                interfaces.append(dev)
        if not interfaces:
            raise oz.OzException.OzException("Could not find interface target device")

        return disks, interfaces

    def _get_disk_and_net_activity(self, libvirt_dom, disks, interfaces):
        """
        Method to collect the disk and network activity by the domain.  The
        method returns two numbers: the first is the sum of all disk activity
        from all disks, and the second is the sum of all network traffic from
        all network devices.
        """
        total_disk_req = 0
        for dev in disks:
            rd_req, rd_bytes, wr_req, wr_bytes, errs = libvirt_dom.blockStats(dev)
            total_disk_req += rd_req + wr_req

        total_net_bytes = 0
        for dev in interfaces:
            rx_bytes, rx_packets, rx_errs, rx_drop, tx_bytes, tx_packets, tx_errs, tx_drop = libvirt_dom.interfaceStats(dev)
            total_net_bytes += rx_bytes + tx_bytes

        return total_disk_req, total_net_bytes

    def _wait_for_install_finish(self, xml, max_time):
        """
        Method to wait for an installation to finish.  This will wait around
        until either the VM has gone away (at which point it is assumed the
        install was successful), or until the timeout is reached (at which
        point it is assumed the install failed and raise an exception).
        """

        libvirt_dom = self.libvirt_conn.createXML(xml, 0)

        disks, interfaces = self._get_disks_and_interfaces(libvirt_dom.XMLDesc(0))

        self.last_disk_activity = 0
        self.last_network_activity = 0
        self.inactivity_countdown = self.inactivity_timeout
        self.saved_exception = None
        if self.has_consolelog:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.settimeout(1)
            self.sock.connect(('127.0.0.1', self.console_listen_port))

        def _finish_cb(self):
            '''
            A callback for _looper to deal with waiting for a guest to finish installing.
            '''
            if self.inactivity_countdown <= 0:
                return True
            try:
                total_disk_req, total_net_bytes = self._get_disk_and_net_activity(libvirt_dom, disks, interfaces)
            except libvirt.libvirtError as e:
                # we save the exception here because we want to raise it later
                # if this was a "real" exception
                self.saved_exception = e
                return True

            # rd_req and wr_req are the *total* number of disk read requests and
            # write requests ever made for this domain.  Similarly rd_bytes and
            # wr_bytes are the total number of network bytes read or written
            # for this domain

            # we define activity as having done a read or write request on the
            # install disk, or having done at least 4KB of network transfers in
            # the last second.  The thinking is that if the installer is putting
            # bits on disk, there will be disk activity, so we should keep
            # waiting.  On the other hand, the installer might be downloading
            # bits to eventually install on disk, so we look for network
            # activity as well.  We say that transfers of at least 4KB must be
            # made, however, to try to reduce false positives from things like
            # ARP requests

            if (total_disk_req == self.last_disk_activity) and (total_net_bytes < (self.last_network_activity + 4096)):
                # if we saw no read or write requests since the last iteration,
                # decrement our activity timer
                self.inactivity_countdown -= 1
            else:
                # if we did see some activity, then we can reset the timer
                self.inactivity_countdown = self.inactivity_timeout

            self.last_disk_activity = total_disk_req
            self.last_network_activity = total_net_bytes

            if self.has_consolelog:
                try:
                    # note that we have to build the data up here, since there
                    # is no guarantee that we will get the whole write in one go
                    data = self.sock.recv(65536)
                    if data:
                        self.log.debug(data.decode('utf-8', errors='surrogateescape'))
{'fraction_non_alphanumeric': 0.09205411487903432, 'fraction_numerical': 0.00791519615364943, 'mean_word_length': 1.9196221906966326, 'pattern_counts': {'":': 22, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 198}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 166, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5149853', 'n_tokens_mistral': 22354, 'n_tokens_neox': 21330, 'n_words': 6139}