<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010 Citrix Systems, Inc.
# Copyright 2011 Piston Cloud Computing, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Helper methods for operations related to the management of VM records and
their attributes like VDIs, VIFs, as well as their lookup functions.
"""

import contextlib
import cPickle as pickle
import decimal
import os
import re
import time
import urllib
import urlparse
import uuid
from xml.dom import minidom
from xml.parsers import expat

from eventlet import greenthread

from nova.compute import instance_types
from nova.compute import power_state
from nova import db
from nova import exception
from nova import flags
from nova.image import glance
from nova.openstack.common import cfg
from nova.openstack.common import excutils
from nova.openstack.common import jsonutils
from nova.openstack.common import log as logging
from nova import utils
from nova.virt.disk import api as disk
from nova.virt import xenapi
from nova.virt.xenapi import volume_utils


LOG = logging.getLogger(__name__)

xenapi_vm_utils_opts = [
    cfg.StrOpt('default_os_type',
               default='linux',
               help='Default OS type'),
    cfg.IntOpt('block_device_creation_timeout',
               default=10,
               help='Time to wait for a block device to be created'),
    cfg.IntOpt('max_kernel_ramdisk_size',
               default=16 * 1024 * 1024,
               help='Maximum size in bytes of kernel or ramdisk images'),
    cfg.StrOpt('sr_matching_filter',
               default='other-config:i18n-key=local-storage',
               help='Filter for finding the SR to be used to install guest '
                    'instances on. The default value is the Local Storage in '
                    'default XenServer/XCP installations. To select an SR '
                    'with a different matching criteria, you could set it to '
                    'other-config:my_favorite_sr=true. On the other hand, to '
                    'fall back on the Default SR, as displayed by XenCenter, '
                    'set this flag to: default-sr:true'),
    cfg.BoolOpt('xenapi_sparse_copy',
                default=True,
                help='Whether to use sparse_copy for copying data on a '
                     'resize down (False will use standard dd). This speeds '
                     'up resizes down considerably since large runs of zeros '
                     'won\'t have to be rsynced'),
    cfg.IntOpt('xenapi_num_vbd_unplug_retries',
               default=10,
               help='Maximum number of retries to unplug VBD'),
    ]

FLAGS = flags.FLAGS
FLAGS.register_opts(xenapi_vm_utils_opts)

XENAPI_POWER_STATE = {
    'Halted': power_state.SHUTDOWN,
    'Running': power_state.RUNNING,
    'Paused': power_state.PAUSED,
    'Suspended': power_state.SUSPENDED,
    'Crashed': power_state.CRASHED}


SECTOR_SIZE = 512
MBR_SIZE_SECTORS = 63
MBR_SIZE_BYTES = MBR_SIZE_SECTORS * SECTOR_SIZE
KERNEL_DIR = '/boot/guest'


class ImageType(object):
    """Enumeration class for distinguishing different image types

    | 0 - kernel image (goes on dom0's filesystem)
    | 1 - ramdisk image (goes on dom0's filesystem)
    | 2 - disk image (local SR, partitioned by objectstore plugin)
    | 3 - raw disk image (local SR, NOT partitioned by plugin)
    | 4 - vhd disk image (local SR, NOT inspected by XS, PV assumed for
    |     linux, HVM assumed for Windows)
    | 5 - ISO disk image (local SR, NOT partitioned by plugin)
    """

    KERNEL = 0
    RAMDISK = 1
    DISK = 2
    DISK_RAW = 3
    DISK_VHD = 4
    DISK_ISO = 5
    _ids = (KERNEL, RAMDISK, DISK, DISK_RAW, DISK_VHD, DISK_ISO)

    KERNEL_STR = "kernel"
    RAMDISK_STR = "ramdisk"
    DISK_STR = "root"
    DISK_RAW_STR = "os_raw"
    DISK_VHD_STR = "vhd"
    DISK_ISO_STR = "iso"
    _strs = (KERNEL_STR, RAMDISK_STR, DISK_STR, DISK_RAW_STR, DISK_VHD_STR,
                DISK_ISO_STR)

    @classmethod
    def to_string(cls, image_type):
        return dict(zip(ImageType._ids, ImageType._strs)).get(image_type)

    @classmethod
    def from_string(cls, image_type_str):
        return dict(zip(ImageType._strs, ImageType._ids)).get(image_type_str)


def create_vm(session, instance, kernel, ramdisk, use_pv_kernel=False):
    """Create a VM record.  Returns new VM reference.
    the use_pv_kernel flag indicates whether the guest is HVM or PV

    There are 3 scenarios:

        1. Using paravirtualization, kernel passed in

        2. Using paravirtualization, kernel within the image

        3. Using hardware virtualization
    """
    inst_type_id = instance.instance_type_id
    instance_type = instance_types.get_instance_type(inst_type_id)
    mem = str(long(instance_type['memory_mb']) * 1024 * 1024)
    vcpus = str(instance_type['vcpus'])

    rec = {
        'actions_after_crash': 'destroy',
        'actions_after_reboot': 'restart',
        'actions_after_shutdown': 'destroy',
        'affinity': '',
        'blocked_operations': {},
        'ha_always_run': False,
        'ha_restart_priority': '',
        'HVM_boot_params': {},
        'HVM_boot_policy': '',
        'is_a_template': False,
        'memory_dynamic_min': mem,
        'memory_dynamic_max': mem,
        'memory_static_min': '0',
        'memory_static_max': mem,
        'memory_target': mem,
        'name_description': '',
        'name_label': instance.name,
        'other_config': {'allowvssprovider': str(False),
                         'nova_uuid': str(instance.uuid)},
        'PCI_bus': '',
        'platform': {'acpi': 'true', 'apic': 'true', 'pae': 'true',
                     'viridian': 'true', 'timeoffset': '0'},
        'PV_args': '',
        'PV_bootloader': '',
        'PV_bootloader_args': '',
        'PV_kernel': '',
        'PV_legacy_args': '',
        'PV_ramdisk': '',
        'recommendations': '',
        'tags': [],
        'user_version': '0',
        'VCPUs_at_startup': vcpus,
        'VCPUs_max': vcpus,
        'VCPUs_params': {},
        'xenstore_data': {}}

    # Complete VM configuration record according to the image type
    # non-raw/raw with PV kernel/raw in HVM mode
    if use_pv_kernel:
        rec['platform']['nx'] = 'false'
        if instance.kernel_id:
            # 1. Kernel explicitly passed in, use that
            rec['PV_args'] = 'root=/dev/xvda1'
            rec['PV_kernel'] = kernel
            rec['PV_ramdisk'] = ramdisk
        else:
            # 2. Use kernel within the image
            rec['PV_bootloader'] = 'pygrub'
    else:
        # 3. Using hardware virtualization
        rec['platform']['nx'] = 'true'
        rec['HVM_boot_params'] = {'order': 'dc'}
        rec['HVM_boot_policy'] = 'BIOS order'

    vm_ref = session.call_xenapi('VM.create', rec)
    LOG.debug(_('Created VM'), instance=instance)
    return vm_ref


def ensure_free_mem(session, instance):
    inst_type_id = instance.instance_type_id
    instance_type = instance_types.get_instance_type(inst_type_id)
    mem = long(instance_type['memory_mb']) * 1024 * 1024
    #get free memory from host
    host = session.get_xenapi_host()
    host_free_mem = long(session.call_xenapi("host.compute_free_memory",
                                             host))
    return host_free_mem >= mem


def find_vbd_by_number(session, vm_ref, number):
    """Get the VBD reference from the device number"""
    vbd_refs = session.call_xenapi("VM.get_VBDs", vm_ref)
    if vbd_refs:
        for vbd_ref in vbd_refs:
            try:
                vbd_rec = session.call_xenapi("VBD.get_record", vbd_ref)
                if vbd_rec['userdevice'] == str(number):
                    return vbd_ref
            except session.XenAPI.Failure, exc:
                LOG.exception(exc)
    raise volume_utils.StorageError(
            _('VBD not found in instance %s') % vm_ref)


def unplug_vbd(session, vbd_ref):
    """Unplug VBD from VM"""
    # Call VBD.unplug on the given VBD, with a retry if we get
    # DEVICE_DETACH_REJECTED.  For reasons which we don't understand,
    # we're seeing the device still in use, even when all processes
    # using the device should be dead.
    max_attempts = FLAGS.xenapi_num_vbd_unplug_retries + 1
    for num_attempt in xrange(1, max_attempts + 1):
        try:
            session.call_xenapi('VBD.unplug', vbd_ref)
            return
        except session.XenAPI.Failure, exc:
            err = len(exc.details) > 0 and exc.details[0]
            if err == 'DEVICE_ALREADY_DETACHED':
                LOG.info(_('VBD %s already detached'), vbd_ref)
                return
            elif err == 'DEVICE_DETACH_REJECTED':
                LOG.info(_('VBD %(vbd_ref)s detach rejected, attempt'
                           ' %(num_attempt)d/%(max_attempts)d'), locals())
            else:
                LOG.exception(exc)
                raise volume_utils.StorageError(
                        _('Unable to unplug VBD %s') % vbd_ref)

        greenthread.sleep(1)

    raise volume_utils.StorageError(
            _('Reached maximum number of retries trying to unplug VBD %s')
            % vbd_ref)


def destroy_vbd(session, vbd_ref):
    """Destroy VBD from host database"""
    try:
        session.call_xenapi('VBD.destroy', vbd_ref)
    except session.XenAPI.Failure, exc:
        LOG.exception(exc)
        raise volume_utils.StorageError(
                _('Unable to destroy VBD %s') % vbd_ref)


def create_vbd(session, vm_ref, vdi_ref, userdevice, vbd_type='disk',
               read_only=False, bootable=False):
    """Create a VBD record and returns its reference."""
    vbd_rec = {}
    vbd_rec['VM'] = vm_ref
    vbd_rec['VDI'] = vdi_ref
    vbd_rec['userdevice'] = str(userdevice)
    vbd_rec['bootable'] = bootable
    vbd_rec['mode'] = read_only and 'RO' or 'RW'
    vbd_rec['type'] = vbd_type
    vbd_rec['unpluggable'] = True
    vbd_rec['empty'] = False
    vbd_rec['other_config'] = {}
    vbd_rec['qos_algorithm_type'] = ''
    vbd_rec['qos_algorithm_params'] = {}
    vbd_rec['qos_supported_algorithms'] = []
    LOG.debug(_('Creating %(vbd_type)s-type VBD for VM %(vm_ref)s,'
                ' VDI %(vdi_ref)s ... '), locals())
    vbd_ref = session.call_xenapi('VBD.create', vbd_rec)
    LOG.debug(_('Created VBD %(vbd_ref)s for VM %(vm_ref)s,'
                ' VDI %(vdi_ref)s.'), locals())
    return vbd_ref


def destroy_vdi(session, vdi_ref):
    try:
        session.call_xenapi('VDI.destroy', vdi_ref)
    except session.XenAPI.Failure, exc:
        LOG.exception(exc)
        raise volume_utils.StorageError(
                _('Unable to destroy VDI %s') % vdi_ref)


def create_vdi(session, sr_ref, info, disk_type, virtual_size,
               read_only=False):
    """Create a VDI record and returns its reference."""
    # create_vdi may be called simply while creating a volume
    # hence information about instance may or may not be present
    otherconf = {}
    if not isinstance(info, basestring):
        name_label = info['name']
        otherconf = {'nova_instance_uuid': info['uuid'],
                     'nova_disk_type': disk_type}
    else:
        name_label = info
    vdi_ref = session.call_xenapi("VDI.create",
         {'name_label': name_label,
          'name_description': disk_type,
          'SR': sr_ref,
          'virtual_size': str(virtual_size),
          'type': 'User',
          'sharable': False,
          'read_only': read_only,
          'xenstore_data': {},
          'other_config': otherconf,
          'sm_config': {},
          'tags': []})
    LOG.debug(_('Created VDI %(vdi_ref)s (%(name_label)s,'
                ' %(virtual_size)s, %(read_only)s) on %(sr_ref)s.'),
              locals())
    return vdi_ref


def copy_vdi(session, sr_ref, vdi_to_copy_ref):
    """Copy a VDI and return the new VDIs reference."""
    vdi_ref = session.call_xenapi('VDI.copy', vdi_to_copy_ref, sr_ref)
    LOG.debug(_('Copied VDI %(vdi_ref)s from VDI '
                '%(vdi_to_copy_ref)s on %(sr_ref)s.') % locals())
    return vdi_ref


def clone_vdi(session, vdi_to_clone_ref):
    """Clones a VDI and return the new VDIs reference."""
    vdi_ref = session.call_xenapi('VDI.clone', vdi_to_clone_ref)
    LOG.debug(_('Cloned VDI %(vdi_ref)s from VDI '
                '%(vdi_to_clone_ref)s') % locals())
    return vdi_ref


def set_vdi_name(session, vdi_uuid, label, description, vdi_ref=None):
    vdi_ref = vdi_ref or session.call_xenapi('VDI.get_by_uuid', vdi_uuid)
    session.call_xenapi('VDI.set_name_label', vdi_ref, label)
    session.call_xenapi('VDI.set_name_description', vdi_ref, description)


def get_vdi_for_vm_safely(session, vm_ref):
    """Retrieves the primary VDI for a VM"""
    vbd_refs = session.call_xenapi("VM.get_VBDs", vm_ref)
    for vbd in vbd_refs:
        vbd_rec = session.call_xenapi("VBD.get_record", vbd)
        # Convention dictates the primary VDI will be userdevice 0
        if vbd_rec['userdevice'] == '0':
            vdi_rec = session.call_xenapi("VDI.get_record", vbd_rec['VDI'])
            return vbd_rec['VDI'], vdi_rec
    raise exception.NovaException(_("No primary VDI found for %(vm_ref)s")
                                  % locals())


def create_snapshot(session, instance, vm_ref, label):
    """Creates Snapshot (Template) VM, Snapshot VBD, Snapshot VDI,
    Snapshot VHD"""
    LOG.debug(_("Snapshotting with label '%(label)s'"), locals(),
              instance=instance)

    vm_vdi_ref, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref)
    sr_ref = vm_vdi_rec["SR"]

    original_parent_uuid = get_vhd_parent_uuid(session, vm_vdi_ref)

    template_vm_ref = session.call_xenapi('VM.snapshot', vm_ref, label)
    template_vdi_rec = get_vdi_for_vm_safely(session, template_vm_ref)[1]
    template_vdi_uuid = template_vdi_rec["uuid"]

    LOG.debug(_('Created snapshot %(template_vm_ref)s'), locals(),
              instance=instance)

    parent_uuid, base_uuid = _wait_for_vhd_coalesce(
            session, instance, sr_ref, vm_vdi_ref, original_parent_uuid)

    template_vdi_uuids = {'base': base_uuid,
                          'image': parent_uuid,
                          'snap': template_vdi_uuid}
    return template_vm_ref, template_vdi_uuids


def get_sr_path(session):
    """Return the path to our storage repository

    This is used when we're dealing with VHDs directly, either by taking
    snapshots or by restoring an image in the DISK_VHD format.
    """
    sr_ref = safe_find_sr(session)
    sr_rec = session.call_xenapi("SR.get_record", sr_ref)
    sr_uuid = sr_rec["uuid"]
    return os.path.join(FLAGS.xenapi_sr_base_path, sr_uuid)


def find_cached_image(session, image_id, sr_ref):
    """Returns the vdi-ref of the cached image."""
    for vdi_ref, vdi_rec in _get_all_vdis_in_sr(session, sr_ref):
        other_config = vdi_rec['other_config']

        try:
            image_id_match = other_config['image-id'] == image_id
        except KeyError:
            image_id_match = False

        # NOTE(sirp): `VDI.copy` stores the partially-completed file in the SR.
        # In order to avoid these half-baked files, we compare its current size
        # to the expected size pulled from the original cache file.
        try:
            size_match = (other_config['expected_physical_utilisation'] ==
                          vdi_rec['physical_utilisation'])
        except KeyError:
            size_match = False

        if image_id_match and size_match:
            return vdi_ref

    return None


def upload_image(context, session, instance, vdi_uuids, image_id):
    """Requests that the Glance plugin bundle the specified VDIs and
    push them into Glance using the specified human-friendly name.
    """
    # NOTE(sirp): Currently we only support uploading images as VHD, there
    # is no RAW equivalent (yet)
    LOG.debug(_("Asking xapi to upload %(vdi_uuids)s as"
                " ID %(image_id)s"), locals(), instance=instance)

    glance_host, glance_port = glance.pick_glance_api_server()

    sys_meta = db.instance_system_metadata_get(context, instance['uuid'])
    properties = {}
    prefix = 'image_'
    for key, value in sys_meta.iteritems():
        if key.startswith(prefix):
            key = key[len(prefix):]
        properties[key] = value
    properties['auto_disk_config'] = instance.auto_disk_config
    properties['os_type'] = instance.os_type or FLAGS.default_os_type

    params = {'vdi_uuids': vdi_uuids,
              'image_id': image_id,
              'glance_host': glance_host,
              'glance_port': glance_port,
              'sr_path': get_sr_path(session),
              'auth_token': getattr(context, 'auth_token', None),
              'properties': properties}

    kwargs = {'params': pickle.dumps(params)}
    session.call_plugin('glance', 'upload_vhd', kwargs)


def resize_disk(session, instance, vdi_ref, instance_type):
    # Copy VDI over to something we can resize
    # NOTE(jerdfelt): Would be nice to just set vdi_ref to read/write
    sr_ref = safe_find_sr(session)
    copy_ref = session.call_xenapi('VDI.copy', vdi_ref, sr_ref)

    try:
        # Resize partition and filesystem down
        auto_configure_disk(session, copy_ref, instance_type['root_gb'])

        # Create new VDI
        vdi_size = instance_type['root_gb'] * 1024 * 1024 * 1024
        new_ref = create_vdi(session, sr_ref, instance, 'root', vdi_size)

        new_uuid = session.call_xenapi('VDI.get_uuid', new_ref)

        # Manually copy contents over
        virtual_size = instance_type['root_gb'] * 1024 * 1024 * 1024
        _copy_partition(session, copy_ref, new_ref, 1, virtual_size)

        return new_ref, new_uuid
    finally:
        destroy_vdi(session, copy_ref)


def auto_configure_disk(session, vdi_ref, new_gb):
    """Partition and resize FS to match the size specified by
    instance_types.root_gb.

    This is a fail-safe to prevent accidentally destroying data on a disk
    erroneously marked as auto_disk_config=True.

    The criteria for allowing resize are:

        1. 'auto_disk_config' must be true for the instance (and image).
           (If we've made it here, then auto_disk_config=True.)

        2. The disk must have only one partition.

        3. The file-system on the one partition must be ext3 or ext4.
    """
    with vdi_attached_here(session, vdi_ref, read_only=False) as dev:
        partitions = _get_partitions(dev)

        if len(partitions) != 1:
            return

        _num, start, old_sectors, ptype = partitions[0]
        if ptype in ('ext3', 'ext4'):
            new_sectors = new_gb * 1024 * 1024 * 1024 / SECTOR_SIZE
            _resize_part_and_fs(dev, start, old_sectors, new_sectors)


def _generate_disk(session, instance, vm_ref, userdevice, name, size_mb,
                   fs_type):
    """
    Steps to programmatically generate a disk:

        1. Create VDI of desired size

        2. Attach VDI to compute worker

        3. Create partition

        4. Create VBD between instance VM and VDI
    """
    # 1. Create VDI
    sr_ref = safe_find_sr(session)
    ONE_MEG = 1024 * 1024
    virtual_size = size_mb * ONE_MEG
    vdi_ref = create_vdi(session, sr_ref, instance, name, virtual_size)

    try:
        # 2. Attach VDI to compute worker (VBD hotplug)
        with vdi_attached_here(session, vdi_ref, read_only=False) as dev:
            # 3. Create partition
            dev_path = utils.make_dev_path(dev)
            utils.execute('parted', '--script', dev_path,
                          'mklabel', 'msdos', run_as_root=True)

            partition_start = 0
            partition_end = size_mb
            utils.execute('parted', '--script', dev_path,
                          'mkpart', 'primary',
                          str(partition_start),
                          str(partition_end),
                          run_as_root=True)

            partition_path = utils.make_dev_path(dev, partition=1)

            if fs_type == 'linux-swap':
                utils.execute('mkswap', partition_path, run_as_root=True)
            elif fs_type is not None:
                utils.execute('mkfs', '-t', fs_type, partition_path,
                              run_as_root=True)

        # 4. Create VBD between instance VM and swap VDI
        create_vbd(session, vm_ref, vdi_ref, userdevice, bootable=False)
    except Exception:
        with excutils.save_and_reraise_exception():
            destroy_vdi(session, vdi_ref)


def generate_swap(session, instance, vm_ref, userdevice, swap_mb):
    # NOTE(jk0): We use a FAT32 filesystem for the Windows swap
    # partition because that is what parted supports.
    is_windows = instance.os_type == "windows"
    fs_type = "vfat" if is_windows else "linux-swap"

    _generate_disk(session, instance, vm_ref, userdevice, 'swap', swap_mb,
                   fs_type)


def generate_ephemeral(session, instance, vm_ref, userdevice, size_gb):
    _generate_disk(session, instance, vm_ref, userdevice, 'ephemeral',
                   size_gb * 1024, FLAGS.default_ephemeral_format)


def create_kernel_image(context, session, instance, image_id, user_id,
                        project_id, image_type):
    """Creates kernel/ramdisk file from the image stored in the cache.
    If the image is not present in the cache, it streams it from glance.

    Returns: A list of dictionaries that describe VDIs
    """
    filename = ""
    if FLAGS.cache_images:
        args = {}
        args['cached-image'] = image_id
        args['new-image-uuid'] = str(uuid.uuid4())
        filename = session.call_plugin(
                'kernel', 'create_kernel_ramdisk', args)

    if filename == "":
        return _fetch_image(context, session, instance, image_id, image_type)
    else:
        vdi_type = ImageType.to_string(image_type)
        return {vdi_type: dict(uuid=None, file=filename)}


def destroy_kernel_ramdisk(session, kernel, ramdisk):
    args = {}
    if kernel:
        args['kernel-file'] = kernel
    if ramdisk:
        args['ramdisk-file'] = ramdisk
    session.call_plugin('kernel', 'remove_kernel_ramdisk', args)


def _create_cached_image(context, session, instance, image_id, image_type):
    sr_ref = safe_find_sr(session)
    sr_type = session.call_xenapi('SR.get_record', sr_ref)["type"]
    vdis = {}

    if FLAGS.use_cow_images and sr_type != "ext":
        LOG.warning(_("Fast cloning is only supported on default local SR "
                      "of type ext. SR on this system was found to be of "
                      "type %(sr_type)s. Ignoring the cow flag.")
                      % locals())

    root_vdi_ref = find_cached_image(session, image_id, sr_ref)
    if root_vdi_ref is None:
        vdis = _fetch_image(context, session, instance, image_id,
                                   image_type)
        root_vdi = vdis['root']
        root_vdi_ref = session.call_xenapi('VDI.get_by_uuid',
                                           root_vdi['uuid'])
        set_vdi_name(session, root_vdi['uuid'], 'Glance Image %s' % image_id,
                     'root', vdi_ref=root_vdi_ref)
        session.call_xenapi('VDI.add_to_other_config',
                            root_vdi_ref, 'image-id', str(image_id))

        for vdi_type, vdi in vdis.iteritems():
            vdi_ref = session.call_xenapi('VDI.get_by_uuid',
                                          vdi['uuid'])

            vdi_rec = session.call_xenapi('VDI.get_record', vdi_ref)
            session.call_xenapi('VDI.add_to_other_config',
                                vdi_ref, 'expected_physical_utilisation',
                                vdi_rec['physical_utilisation'])

            if vdi_type == 'swap':
                session.call_xenapi('VDI.add_to_other_config',
                                    root_vdi_ref, 'swap-disk',
                                    str(vdi['uuid']))

    if FLAGS.use_cow_images and sr_type == 'ext':
        new_vdi_ref = clone_vdi(session, root_vdi_ref)
    else:
        new_vdi_ref = copy_vdi(session, sr_ref, root_vdi_ref)

    # Set the name label for the image we just created and remove image id
    # field from other-config.
    session.call_xenapi('VDI.remove_from_other_config',
                        new_vdi_ref, 'image-id')

    vdi_type = ("root" if image_type == ImageType.DISK_VHD
                else ImageType.to_string(image_type))
    vdi_uuid = session.call_xenapi('VDI.get_uuid', new_vdi_ref)
    vdis[vdi_type] = dict(uuid=vdi_uuid, file=None)

    # Create a swap disk if the glance image had one associated with it.
    vdi_rec = session.call_xenapi('VDI.get_record', root_vdi_ref)
    if 'swap-disk' in vdi_rec['other_config']:
        swap_disk_uuid = vdi_rec['other_config']['swap-disk']
        swap_vdi_ref = session.call_xenapi('VDI.get_by_uuid',
                                           swap_disk_uuid)
        new_swap_vdi_ref = copy_vdi(session, sr_ref, swap_vdi_ref)
        new_swap_vdi_uuid = session.call_xenapi('VDI.get_uuid',
                                                new_swap_vdi_ref)
        vdis['swap'] = dict(uuid=new_swap_vdi_uuid, file=None)

    return vdis


def create_image(context, session, instance, image_id, image_type):
    """Creates VDI from the image stored in the local cache. If the image
    is not present in the cache, it streams it from glance.

    Returns: A list of dictionaries that describe VDIs
    """
    cache_images = FLAGS.cache_images.lower()

    # Deterimine if the image is cacheable
    if image_type == ImageType.DISK_ISO:
        cache = False
    elif cache_images == 'all':
        cache = True
    elif cache_images == 'some':
        # FIXME(sirp): This should be eager loaded like instance metadata
        sys_meta = db.instance_system_metadata_get(context,
                instance['uuid'])
        try:
            cache = utils.bool_from_str(sys_meta['image_cache_in_nova'])
        except KeyError:
            cache = False
    elif cache_images == 'none':
        cache = False
    else:
        LOG.warning(_("Unrecognized cache_images value '%s', defaulting to"
                      " True"), FLAGS.cache_images)
        cache = True

    # Fetch (and cache) the image
    if cache:
        vdis = _create_cached_image(
                context, session, instance, image_id, image_type)
    else:
        vdis = _fetch_image(
                context, session, instance, image_id, image_type)

    # Set the name label and description to easily identify what
    # instance and disk it's for
    for vdi_type, vdi in vdis.iteritems():
        set_vdi_name(session, vdi['uuid'], instance.name, vdi_type)

    return vdis


def _fetch_image(context, session, instance, image_id, image_type):
    """Fetch image from glance based on image type.

    Returns: A single filename if image_type is KERNEL or RAMDISK
             A list of dictionaries that describe VDIs, otherwise
    """
    if image_type == ImageType.DISK_VHD:
        vdis = _fetch_vhd_image(context, session, instance, image_id)
    else:
        vdis = _fetch_disk_image(context, session, instance, image_id,
                                 image_type)

    for vdi_type, vdi in vdis.iteritems():
        vdi_uuid = vdi['uuid']
        LOG.debug(_("Fetched VDIs of type '%(vdi_type)s' with UUID"
                    "  '%(vdi_uuid)s'"),
                  locals(), instance=instance)

    return vdis


def _fetch_using_dom0_plugin_with_retry(context, session, image_id,
                                        plugin_name, params, callback=None):
    max_attempts = FLAGS.glance_num_retries + 1<commit_msg>Fetch kernel/ramdisk images directly<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010 Citrix Systems, Inc.
# Copyright 2011 Piston Cloud Computing, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Helper methods for operations related to the management of VM records and
their attributes like VDIs, VIFs, as well as their lookup functions.
"""

import contextlib
import cPickle as pickle
import decimal
import os
import re
import time
import urllib
import urlparse
import uuid
from xml.dom import minidom
from xml.parsers import expat

from eventlet import greenthread

from nova.compute import instance_types
from nova.compute import power_state
from nova import db
from nova import exception
from nova import flags
from nova.image import glance
from nova.openstack.common import cfg
from nova.openstack.common import excutils
from nova.openstack.common import jsonutils
from nova.openstack.common import log as logging
from nova import utils
from nova.virt.disk import api as disk
from nova.virt import xenapi
from nova.virt.xenapi import volume_utils


LOG = logging.getLogger(__name__)

xenapi_vm_utils_opts = [
    cfg.StrOpt('default_os_type',
               default='linux',
               help='Default OS type'),
    cfg.IntOpt('block_device_creation_timeout',
               default=10,
               help='Time to wait for a block device to be created'),
    cfg.IntOpt('max_kernel_ramdisk_size',
               default=16 * 1024 * 1024,
               help='Maximum size in bytes of kernel or ramdisk images'),
    cfg.StrOpt('sr_matching_filter',
               default='other-config:i18n-key=local-storage',
               help='Filter for finding the SR to be used to install guest '
                    'instances on. The default value is the Local Storage in '
                    'default XenServer/XCP installations. To select an SR '
                    'with a different matching criteria, you could set it to '
                    'other-config:my_favorite_sr=true. On the other hand, to '
                    'fall back on the Default SR, as displayed by XenCenter, '
                    'set this flag to: default-sr:true'),
    cfg.BoolOpt('xenapi_sparse_copy',
                default=True,
                help='Whether to use sparse_copy for copying data on a '
                     'resize down (False will use standard dd). This speeds '
                     'up resizes down considerably since large runs of zeros '
                     'won\'t have to be rsynced'),
    cfg.IntOpt('xenapi_num_vbd_unplug_retries',
               default=10,
               help='Maximum number of retries to unplug VBD'),
    ]

FLAGS = flags.FLAGS
FLAGS.register_opts(xenapi_vm_utils_opts)

XENAPI_POWER_STATE = {
    'Halted': power_state.SHUTDOWN,
    'Running': power_state.RUNNING,
    'Paused': power_state.PAUSED,
    'Suspended': power_state.SUSPENDED,
    'Crashed': power_state.CRASHED}


SECTOR_SIZE = 512
MBR_SIZE_SECTORS = 63
MBR_SIZE_BYTES = MBR_SIZE_SECTORS * SECTOR_SIZE
KERNEL_DIR = '/boot/guest'


class ImageType(object):
    """Enumeration class for distinguishing different image types

    | 0 - kernel image (goes on dom0's filesystem)
    | 1 - ramdisk image (goes on dom0's filesystem)
    | 2 - disk image (local SR, partitioned by objectstore plugin)
    | 3 - raw disk image (local SR, NOT partitioned by plugin)
    | 4 - vhd disk image (local SR, NOT inspected by XS, PV assumed for
    |     linux, HVM assumed for Windows)
    | 5 - ISO disk image (local SR, NOT partitioned by plugin)
    """

    KERNEL = 0
    RAMDISK = 1
    DISK = 2
    DISK_RAW = 3
    DISK_VHD = 4
    DISK_ISO = 5
    _ids = (KERNEL, RAMDISK, DISK, DISK_RAW, DISK_VHD, DISK_ISO)

    KERNEL_STR = "kernel"
    RAMDISK_STR = "ramdisk"
    DISK_STR = "root"
    DISK_RAW_STR = "os_raw"
    DISK_VHD_STR = "vhd"
    DISK_ISO_STR = "iso"
    _strs = (KERNEL_STR, RAMDISK_STR, DISK_STR, DISK_RAW_STR, DISK_VHD_STR,
                DISK_ISO_STR)

    @classmethod
    def to_string(cls, image_type):
        return dict(zip(ImageType._ids, ImageType._strs)).get(image_type)

    @classmethod
    def from_string(cls, image_type_str):
        return dict(zip(ImageType._strs, ImageType._ids)).get(image_type_str)


def create_vm(session, instance, kernel, ramdisk, use_pv_kernel=False):
    """Create a VM record.  Returns new VM reference.
    the use_pv_kernel flag indicates whether the guest is HVM or PV

    There are 3 scenarios:

        1. Using paravirtualization, kernel passed in

        2. Using paravirtualization, kernel within the image

        3. Using hardware virtualization
    """
    inst_type_id = instance.instance_type_id
    instance_type = instance_types.get_instance_type(inst_type_id)
    mem = str(long(instance_type['memory_mb']) * 1024 * 1024)
    vcpus = str(instance_type['vcpus'])

    rec = {
        'actions_after_crash': 'destroy',
        'actions_after_reboot': 'restart',
        'actions_after_shutdown': 'destroy',
        'affinity': '',
        'blocked_operations': {},
        'ha_always_run': False,
        'ha_restart_priority': '',
        'HVM_boot_params': {},
        'HVM_boot_policy': '',
        'is_a_template': False,
        'memory_dynamic_min': mem,
        'memory_dynamic_max': mem,
        'memory_static_min': '0',
        'memory_static_max': mem,
        'memory_target': mem,
        'name_description': '',
        'name_label': instance.name,
        'other_config': {'allowvssprovider': str(False),
                         'nova_uuid': str(instance.uuid)},
        'PCI_bus': '',
        'platform': {'acpi': 'true', 'apic': 'true', 'pae': 'true',
                     'viridian': 'true', 'timeoffset': '0'},
        'PV_args': '',
        'PV_bootloader': '',
        'PV_bootloader_args': '',
        'PV_kernel': '',
        'PV_legacy_args': '',
        'PV_ramdisk': '',
        'recommendations': '',
        'tags': [],
        'user_version': '0',
        'VCPUs_at_startup': vcpus,
        'VCPUs_max': vcpus,
        'VCPUs_params': {},
        'xenstore_data': {}}

    # Complete VM configuration record according to the image type
    # non-raw/raw with PV kernel/raw in HVM mode
    if use_pv_kernel:
        rec['platform']['nx'] = 'false'
        if instance.kernel_id:
            # 1. Kernel explicitly passed in, use that
            rec['PV_args'] = 'root=/dev/xvda1'
            rec['PV_kernel'] = kernel
            rec['PV_ramdisk'] = ramdisk
        else:
            # 2. Use kernel within the image
            rec['PV_bootloader'] = 'pygrub'
    else:
        # 3. Using hardware virtualization
        rec['platform']['nx'] = 'true'
        rec['HVM_boot_params'] = {'order': 'dc'}
        rec['HVM_boot_policy'] = 'BIOS order'

    vm_ref = session.call_xenapi('VM.create', rec)
    LOG.debug(_('Created VM'), instance=instance)
    return vm_ref


def ensure_free_mem(session, instance):
    inst_type_id = instance.instance_type_id
    instance_type = instance_types.get_instance_type(inst_type_id)
    mem = long(instance_type['memory_mb']) * 1024 * 1024
    #get free memory from host
    host = session.get_xenapi_host()
    host_free_mem = long(session.call_xenapi("host.compute_free_memory",
                                             host))
    return host_free_mem >= mem


def find_vbd_by_number(session, vm_ref, number):
    """Get the VBD reference from the device number"""
    vbd_refs = session.call_xenapi("VM.get_VBDs", vm_ref)
    if vbd_refs:
        for vbd_ref in vbd_refs:
            try:
                vbd_rec = session.call_xenapi("VBD.get_record", vbd_ref)
                if vbd_rec['userdevice'] == str(number):
                    return vbd_ref
            except session.XenAPI.Failure, exc:
                LOG.exception(exc)
    raise volume_utils.StorageError(
            _('VBD not found in instance %s') % vm_ref)


def unplug_vbd(session, vbd_ref):
    """Unplug VBD from VM"""
    # Call VBD.unplug on the given VBD, with a retry if we get
    # DEVICE_DETACH_REJECTED.  For reasons which we don't understand,
    # we're seeing the device still in use, even when all processes
    # using the device should be dead.
    max_attempts = FLAGS.xenapi_num_vbd_unplug_retries + 1
    for num_attempt in xrange(1, max_attempts + 1):
        try:
            session.call_xenapi('VBD.unplug', vbd_ref)
            return
        except session.XenAPI.Failure, exc:
            err = len(exc.details) > 0 and exc.details[0]
            if err == 'DEVICE_ALREADY_DETACHED':
                LOG.info(_('VBD %s already detached'), vbd_ref)
                return
            elif err == 'DEVICE_DETACH_REJECTED':
                LOG.info(_('VBD %(vbd_ref)s detach rejected, attempt'
                           ' %(num_attempt)d/%(max_attempts)d'), locals())
            else:
                LOG.exception(exc)
                raise volume_utils.StorageError(
                        _('Unable to unplug VBD %s') % vbd_ref)

        greenthread.sleep(1)

    raise volume_utils.StorageError(
            _('Reached maximum number of retries trying to unplug VBD %s')
            % vbd_ref)


def destroy_vbd(session, vbd_ref):
    """Destroy VBD from host database"""
    try:
        session.call_xenapi('VBD.destroy', vbd_ref)
    except session.XenAPI.Failure, exc:
        LOG.exception(exc)
        raise volume_utils.StorageError(
                _('Unable to destroy VBD %s') % vbd_ref)


def create_vbd(session, vm_ref, vdi_ref, userdevice, vbd_type='disk',
               read_only=False, bootable=False):
    """Create a VBD record and returns its reference."""
    vbd_rec = {}
    vbd_rec['VM'] = vm_ref
    vbd_rec['VDI'] = vdi_ref
    vbd_rec['userdevice'] = str(userdevice)
    vbd_rec['bootable'] = bootable
    vbd_rec['mode'] = read_only and 'RO' or 'RW'
    vbd_rec['type'] = vbd_type
    vbd_rec['unpluggable'] = True
    vbd_rec['empty'] = False
    vbd_rec['other_config'] = {}
    vbd_rec['qos_algorithm_type'] = ''
    vbd_rec['qos_algorithm_params'] = {}
    vbd_rec['qos_supported_algorithms'] = []
    LOG.debug(_('Creating %(vbd_type)s-type VBD for VM %(vm_ref)s,'
                ' VDI %(vdi_ref)s ... '), locals())
    vbd_ref = session.call_xenapi('VBD.create', vbd_rec)
    LOG.debug(_('Created VBD %(vbd_ref)s for VM %(vm_ref)s,'
                ' VDI %(vdi_ref)s.'), locals())
    return vbd_ref


def destroy_vdi(session, vdi_ref):
    try:
        session.call_xenapi('VDI.destroy', vdi_ref)
    except session.XenAPI.Failure, exc:
        LOG.exception(exc)
        raise volume_utils.StorageError(
                _('Unable to destroy VDI %s') % vdi_ref)


def create_vdi(session, sr_ref, info, disk_type, virtual_size,
               read_only=False):
    """Create a VDI record and returns its reference."""
    # create_vdi may be called simply while creating a volume
    # hence information about instance may or may not be present
    otherconf = {}
    if not isinstance(info, basestring):
        name_label = info['name']
        otherconf = {'nova_instance_uuid': info['uuid'],
                     'nova_disk_type': disk_type}
    else:
        name_label = info
    vdi_ref = session.call_xenapi("VDI.create",
         {'name_label': name_label,
          'name_description': disk_type,
          'SR': sr_ref,
          'virtual_size': str(virtual_size),
          'type': 'User',
          'sharable': False,
          'read_only': read_only,
          'xenstore_data': {},
          'other_config': otherconf,
          'sm_config': {},
          'tags': []})
    LOG.debug(_('Created VDI %(vdi_ref)s (%(name_label)s,'
                ' %(virtual_size)s, %(read_only)s) on %(sr_ref)s.'),
              locals())
    return vdi_ref


def copy_vdi(session, sr_ref, vdi_to_copy_ref):
    """Copy a VDI and return the new VDIs reference."""
    vdi_ref = session.call_xenapi('VDI.copy', vdi_to_copy_ref, sr_ref)
    LOG.debug(_('Copied VDI %(vdi_ref)s from VDI '
                '%(vdi_to_copy_ref)s on %(sr_ref)s.') % locals())
    return vdi_ref


def clone_vdi(session, vdi_to_clone_ref):
    """Clones a VDI and return the new VDIs reference."""
    vdi_ref = session.call_xenapi('VDI.clone', vdi_to_clone_ref)
    LOG.debug(_('Cloned VDI %(vdi_ref)s from VDI '
                '%(vdi_to_clone_ref)s') % locals())
    return vdi_ref


def set_vdi_name(session, vdi_uuid, label, description, vdi_ref=None):
    vdi_ref = vdi_ref or session.call_xenapi('VDI.get_by_uuid', vdi_uuid)
    session.call_xenapi('VDI.set_name_label', vdi_ref, label)
    session.call_xenapi('VDI.set_name_description', vdi_ref, description)


def get_vdi_for_vm_safely(session, vm_ref):
    """Retrieves the primary VDI for a VM"""
    vbd_refs = session.call_xenapi("VM.get_VBDs", vm_ref)
    for vbd in vbd_refs:
        vbd_rec = session.call_xenapi("VBD.get_record", vbd)
        # Convention dictates the primary VDI will be userdevice 0
        if vbd_rec['userdevice'] == '0':
            vdi_rec = session.call_xenapi("VDI.get_record", vbd_rec['VDI'])
            return vbd_rec['VDI'], vdi_rec
    raise exception.NovaException(_("No primary VDI found for %(vm_ref)s")
                                  % locals())


def create_snapshot(session, instance, vm_ref, label):
    """Creates Snapshot (Template) VM, Snapshot VBD, Snapshot VDI,
    Snapshot VHD"""
    LOG.debug(_("Snapshotting with label '%(label)s'"), locals(),
              instance=instance)

    vm_vdi_ref, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref)
    sr_ref = vm_vdi_rec["SR"]

    original_parent_uuid = get_vhd_parent_uuid(session, vm_vdi_ref)

    template_vm_ref = session.call_xenapi('VM.snapshot', vm_ref, label)
    template_vdi_rec = get_vdi_for_vm_safely(session, template_vm_ref)[1]
    template_vdi_uuid = template_vdi_rec["uuid"]

    LOG.debug(_('Created snapshot %(template_vm_ref)s'), locals(),
              instance=instance)

    parent_uuid, base_uuid = _wait_for_vhd_coalesce(
            session, instance, sr_ref, vm_vdi_ref, original_parent_uuid)

    template_vdi_uuids = {'base': base_uuid,
                          'image': parent_uuid,
                          'snap': template_vdi_uuid}
    return template_vm_ref, template_vdi_uuids


def get_sr_path(session):
    """Return the path to our storage repository

    This is used when we're dealing with VHDs directly, either by taking
    snapshots or by restoring an image in the DISK_VHD format.
    """
    sr_ref = safe_find_sr(session)
    sr_rec = session.call_xenapi("SR.get_record", sr_ref)
    sr_uuid = sr_rec["uuid"]
    return os.path.join(FLAGS.xenapi_sr_base_path, sr_uuid)


def find_cached_image(session, image_id, sr_ref):
    """Returns the vdi-ref of the cached image."""
    for vdi_ref, vdi_rec in _get_all_vdis_in_sr(session, sr_ref):
        other_config = vdi_rec['other_config']

        try:
            image_id_match = other_config['image-id'] == image_id
        except KeyError:
            image_id_match = False

        # NOTE(sirp): `VDI.copy` stores the partially-completed file in the SR.
        # In order to avoid these half-baked files, we compare its current size
        # to the expected size pulled from the original cache file.
        try:
            size_match = (other_config['expected_physical_utilisation'] ==
                          vdi_rec['physical_utilisation'])
        except KeyError:
            size_match = False

        if image_id_match and size_match:
            return vdi_ref

    return None


def upload_image(context, session, instance, vdi_uuids, image_id):
    """Requests that the Glance plugin bundle the specified VDIs and
    push them into Glance using the specified human-friendly name.
    """
    # NOTE(sirp): Currently we only support uploading images as VHD, there
    # is no RAW equivalent (yet)
    LOG.debug(_("Asking xapi to upload %(vdi_uuids)s as"
                " ID %(image_id)s"), locals(), instance=instance)

    glance_host, glance_port = glance.pick_glance_api_server()

    sys_meta = db.instance_system_metadata_get(context, instance['uuid'])
    properties = {}
    prefix = 'image_'
    for key, value in sys_meta.iteritems():
        if key.startswith(prefix):
            key = key[len(prefix):]
        properties[key] = value
    properties['auto_disk_config'] = instance.auto_disk_config
    properties['os_type'] = instance.os_type or FLAGS.default_os_type

    params = {'vdi_uuids': vdi_uuids,
              'image_id': image_id,
              'glance_host': glance_host,
              'glance_port': glance_port,
              'sr_path': get_sr_path(session),
              'auth_token': getattr(context, 'auth_token', None),
              'properties': properties}

    kwargs = {'params': pickle.dumps(params)}
    session.call_plugin('glance', 'upload_vhd', kwargs)


def resize_disk(session, instance, vdi_ref, instance_type):
    # Copy VDI over to something we can resize
    # NOTE(jerdfelt): Would be nice to just set vdi_ref to read/write
    sr_ref = safe_find_sr(session)
    copy_ref = session.call_xenapi('VDI.copy', vdi_ref, sr_ref)

    try:
        # Resize partition and filesystem down
        auto_configure_disk(session, copy_ref, instance_type['root_gb'])

        # Create new VDI
        vdi_size = instance_type['root_gb'] * 1024 * 1024 * 1024
        new_ref = create_vdi(session, sr_ref, instance, 'root', vdi_size)

        new_uuid = session.call_xenapi('VDI.get_uuid', new_ref)

        # Manually copy contents over
        virtual_size = instance_type['root_gb'] * 1024 * 1024 * 1024
        _copy_partition(session, copy_ref, new_ref, 1, virtual_size)

        return new_ref, new_uuid
    finally:
        destroy_vdi(session, copy_ref)


def auto_configure_disk(session, vdi_ref, new_gb):
    """Partition and resize FS to match the size specified by
    instance_types.root_gb.

    This is a fail-safe to prevent accidentally destroying data on a disk
    erroneously marked as auto_disk_config=True.

    The criteria for allowing resize are:

        1. 'auto_disk_config' must be true for the instance (and image).
           (If we've made it here, then auto_disk_config=True.)

        2. The disk must have only one partition.

        3. The file-system on the one partition must be ext3 or ext4.
    """
    with vdi_attached_here(session, vdi_ref, read_only=False) as dev:
        partitions = _get_partitions(dev)

        if len(partitions) != 1:
            return

        _num, start, old_sectors, ptype = partitions[0]
        if ptype in ('ext3', 'ext4'):
            new_sectors = new_gb * 1024 * 1024 * 1024 / SECTOR_SIZE
            _resize_part_and_fs(dev, start, old_sectors, new_sectors)


def _generate_disk(session, instance, vm_ref, userdevice, name, size_mb,
                   fs_type):
    """
    Steps to programmatically generate a disk:

        1. Create VDI of desired size

        2. Attach VDI to compute worker

        3. Create partition

        4. Create VBD between instance VM and VDI
    """
    # 1. Create VDI
    sr_ref = safe_find_sr(session)
    ONE_MEG = 1024 * 1024
    virtual_size = size_mb * ONE_MEG
    vdi_ref = create_vdi(session, sr_ref, instance, name, virtual_size)

    try:
        # 2. Attach VDI to compute worker (VBD hotplug)
        with vdi_attached_here(session, vdi_ref, read_only=False) as dev:
            # 3. Create partition
            dev_path = utils.make_dev_path(dev)
            utils.execute('parted', '--script', dev_path,
                          'mklabel', 'msdos', run_as_root=True)

            partition_start = 0
            partition_end = size_mb
            utils.execute('parted', '--script', dev_path,
                          'mkpart', 'primary',
                          str(partition_start),
                          str(partition_end),
                          run_as_root=True)

            partition_path = utils.make_dev_path(dev, partition=1)

            if fs_type == 'linux-swap':
                utils.execute('mkswap', partition_path, run_as_root=True)
            elif fs_type is not None:
                utils.execute('mkfs', '-t', fs_type, partition_path,
                              run_as_root=True)

        # 4. Create VBD between instance VM and swap VDI
        create_vbd(session, vm_ref, vdi_ref, userdevice, bootable=False)
    except Exception:
        with excutils.save_and_reraise_exception():
            destroy_vdi(session, vdi_ref)


def generate_swap(session, instance, vm_ref, userdevice, swap_mb):
    # NOTE(jk0): We use a FAT32 filesystem for the Windows swap
    # partition because that is what parted supports.
    is_windows = instance.os_type == "windows"
    fs_type = "vfat" if is_windows else "linux-swap"

    _generate_disk(session, instance, vm_ref, userdevice, 'swap', swap_mb,
                   fs_type)


def generate_ephemeral(session, instance, vm_ref, userdevice, size_gb):
    _generate_disk(session, instance, vm_ref, userdevice, 'ephemeral',
                   size_gb * 1024, FLAGS.default_ephemeral_format)


def create_kernel_image(context, session, instance, image_id, user_id,
                        project_id, image_type):
    """Creates kernel/ramdisk file from the image stored in the cache.
    If the image is not present in the cache, it streams it from glance.

    Returns: A list of dictionaries that describe VDIs
    """
    filename = ""
    if FLAGS.cache_images:
        args = {}
        args['cached-image'] = image_id
        args['new-image-uuid'] = str(uuid.uuid4())
        filename = session.call_plugin('kernel', 'create_kernel_ramdisk', args)

    if filename == "":
        return _fetch_disk_image(context, session, instance, image_id,
                                 image_type)
    else:
        vdi_type = ImageType.to_string(image_type)
        return {vdi_type: dict(uuid=None, file=filename)}


def destroy_kernel_ramdisk(session, kernel, ramdisk):
    args = {}
    if kernel:
        args['kernel-file'] = kernel
    if ramdisk:
        args['ramdisk-file'] = ramdisk
    session.call_plugin('kernel', 'remove_kernel_ramdisk', args)


def _create_cached_image(context, session, instance, image_id, image_type):
    sr_ref = safe_find_sr(session)
    sr_type = session.call_xenapi('SR.get_record', sr_ref)["type"]
    vdis = {}

    if FLAGS.use_cow_images and sr_type != "ext":
        LOG.warning(_("Fast cloning is only supported on default local SR "
                      "of type ext. SR on this system was found to be of "
                      "type %(sr_type)s. Ignoring the cow flag.")
                      % locals())

    root_vdi_ref = find_cached_image(session, image_id, sr_ref)
    if root_vdi_ref is None:
        vdis = _fetch_image(context, session, instance, image_id, image_type)
        root_vdi = vdis['root']
        root_vdi_ref = session.call_xenapi('VDI.get_by_uuid',
                                           root_vdi['uuid'])
        set_vdi_name(session, root_vdi['uuid'], 'Glance Image %s' % image_id,
                     'root', vdi_ref=root_vdi_ref)
        session.call_xenapi('VDI.add_to_other_config',
                            root_vdi_ref, 'image-id', str(image_id))

        for vdi_type, vdi in vdis.iteritems():
            vdi_ref = session.call_xenapi('VDI.get_by_uuid',
                                          vdi['uuid'])

            vdi_rec = session.call_xenapi('VDI.get_record', vdi_ref)
            session.call_xenapi('VDI.add_to_other_config',
                                vdi_ref, 'expected_physical_utilisation',
                                vdi_rec['physical_utilisation'])

            if vdi_type == 'swap':
                session.call_xenapi('VDI.add_to_other_config',
                                    root_vdi_ref, 'swap-disk',
                                    str(vdi['uuid']))

    if FLAGS.use_cow_images and sr_type == 'ext':
        new_vdi_ref = clone_vdi(session, root_vdi_ref)
    else:
        new_vdi_ref = copy_vdi(session, sr_ref, root_vdi_ref)

    # Set the name label for the image we just created and remove image id
    # field from other-config.
    session.call_xenapi('VDI.remove_from_other_config',
                        new_vdi_ref, 'image-id')

    vdi_type = ("root" if image_type == ImageType.DISK_VHD
                else ImageType.to_string(image_type))
    vdi_uuid = session.call_xenapi('VDI.get_uuid', new_vdi_ref)
    vdis[vdi_type] = dict(uuid=vdi_uuid, file=None)

    # Create a swap disk if the glance image had one associated with it.
    vdi_rec = session.call_xenapi('VDI.get_record', root_vdi_ref)
    if 'swap-disk' in vdi_rec['other_config']:
        swap_disk_uuid = vdi_rec['other_config']['swap-disk']
        swap_vdi_ref = session.call_xenapi('VDI.get_by_uuid',
                                           swap_disk_uuid)
        new_swap_vdi_ref = copy_vdi(session, sr_ref, swap_vdi_ref)
        new_swap_vdi_uuid = session.call_xenapi('VDI.get_uuid',
                                                new_swap_vdi_ref)
        vdis['swap'] = dict(uuid=new_swap_vdi_uuid, file=None)

    return vdis


def create_image(context, session, instance, image_id, image_type):
    """Creates VDI from the image stored in the local cache. If the image
    is not present in the cache, it streams it from glance.

    Returns: A list of dictionaries that describe VDIs
    """
    cache_images = FLAGS.cache_images.lower()

    # Deterimine if the image is cacheable
    if image_type == ImageType.DISK_ISO:
        cache = False
    elif cache_images == 'all':
        cache = True
    elif cache_images == 'some':
        # FIXME(sirp): This should be eager loaded like instance metadata
        sys_meta = db.instance_system_metadata_get(context,
                instance['uuid'])
        try:
            cache = utils.bool_from_str(sys_meta['image_cache_in_nova'])
        except KeyError:
            cache = False
    elif cache_images == 'none':
        cache = False
    else:
        LOG.warning(_("Unrecognized cache_images value '%s', defaulting to"
                      " True"), FLAGS.cache_images)
        cache = True

    # Fetch (and cache) the image
    if cache:
        vdis = _create_cached_image(
                context, session, instance, image_id, image_type)
    else:
        vdis = _fetch_image(
                context, session, instance, image_id, image_type)

    # Set the name label and description to easily identify what
    # instance and disk it's for
    for vdi_type, vdi in vdis.iteritems():
        set_vdi_name(session, vdi['uuid'], instance.name, vdi_type)

    return vdis


def _fetch_image(context, session, instance, image_id, image_type):
    """Fetch image from glance based on image type.

    Returns: A single filename if image_type is KERNEL or RAMDISK
             A list of dictionaries that describe VDIs, otherwise
    """
    if image_type == ImageType.DISK_VHD:
        vdis = _fetch_vhd_image(context, session, instance, image_id)
    else:
        vdis = _fetch_disk_image(context, session, instance, image_id,
                                 image_type)

    for vdi_type, vdi in vdis.iteritems():
        vdi_uuid = vdi['uuid']
        LOG.debug(_("Fetched VDIs of type '%(vdi_type)s' with UUID"
                    " '%(vdi_uuid)s'"),
                  locals(), instance=instance)

    return vdis


def _fetch_using_dom0_plugin_with_retry(context, session, image_id,
                                        plugin_name, params, callback=None):
    max_attempts = FLAGS.glance_num_retries + 1
{'fraction_non_alphanumeric': 0.09926797199815217, 'fraction_numerical': 0.005685654383284176, 'mean_word_length': 2.5440463446886215, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2734092', 'n_tokens_mistral': 18349, 'n_tokens_neox': 17562, 'n_words': 4549}