<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright (c) 2010 Citrix Systems, Inc.
# Copyright (c) 2011 Piston Cloud Computing, Inc
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
A connection to a hypervisor through libvirt.

Supports KVM, LXC, QEMU, UML, and XEN.

**Related Flags**

:libvirt_type:  Libvirt domain type.  Can be kvm, qemu, uml, xen
                (default: kvm).
:libvirt_uri:  Override for the default libvirt URI (depends on libvirt_type).
:libvirt_xml_template:  Libvirt XML Template.
:rescue_image_id:  Rescue ami image (None = original image).
:rescue_kernel_id:  Rescue aki image (None = original image).
:rescue_ramdisk_id:  Rescue ari image (None = original image).
:injected_network_template:  Template file for injected network
:allow_same_net_traffic:  Whether to allow in project network traffic

"""

import hashlib
import functools
import multiprocessing
import netaddr
import os
import random
import re
import shutil
import sys
import tempfile
import time
import uuid
import subprocess
from xml.dom import minidom
from xml.etree import ElementTree

from eventlet import greenthread
from eventlet import tpool

from nova import block_device
from nova import context as nova_context
from nova import db
from nova import exception
from nova import flags
import nova.image
from nova import log as logging
from nova import utils
from nova import vnc
from nova.auth import manager
from nova.compute import instance_types
from nova.compute import power_state
from nova.compute import vm_states
from nova.virt import disk
from nova.virt import driver
from nova.virt import images
from nova.virt.libvirt import netutils


libvirt = None
libxml2 = None
Template = None

LOG = logging.getLogger('nova.virt.libvirt_conn')

#ISI
lxc_mounts = {}
#!ISI

FLAGS = flags.FLAGS
flags.DECLARE('live_migration_retry_count', 'nova.compute.manager')
# TODO(vish): These flags should probably go into a shared location
flags.DEFINE_string('rescue_image_id', None, 'Rescue ami image')
flags.DEFINE_string('rescue_kernel_id', None, 'Rescue aki image')
flags.DEFINE_string('rescue_ramdisk_id', None, 'Rescue ari image')
flags.DEFINE_string('libvirt_xml_template',
                    utils.abspath('virt/libvirt.xml.template'),
                    'Libvirt XML Template')
flags.DEFINE_string('libvirt_type',
                    'kvm',
                    'Libvirt domain type (valid options are: '
                    'kvm, lxc, qemu, uml, xen)')
flags.DEFINE_string('libvirt_uri',
                    '',
                    'Override the default libvirt URI (which is dependent'
                    ' on libvirt_type)')
flags.DEFINE_bool('allow_same_net_traffic',
                  True,
                  'Whether to allow network traffic from same network')
flags.DEFINE_bool('use_cow_images',
                  True,
                  'Whether to use cow images')
flags.DEFINE_string('ajaxterm_portrange',
                    '10000-12000',
                    'Range of ports that ajaxterm should randomly try to bind')
flags.DEFINE_string('firewall_driver',
                    'nova.virt.libvirt.firewall.IptablesFirewallDriver',
                    'Firewall driver (defaults to iptables)')
flags.DEFINE_string('cpuinfo_xml_template',
                    utils.abspath('virt/cpuinfo.xml.template'),
                    'CpuInfo XML Template (Used only live migration now)')
flags.DEFINE_string('live_migration_uri',
                    "qemu+tcp://%s/system",
                    'Define protocol used by live_migration feature')
flags.DEFINE_string('live_migration_flag',
                    "VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER",
                    'Define live migration behavior.')
flags.DEFINE_string('block_migration_flag',
                    "VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER, "
                    "VIR_MIGRATE_NON_SHARED_INC",
                    'Define block migration behavior.')
flags.DEFINE_integer('live_migration_bandwidth', 0,
                    'Define live migration behavior')
flags.DEFINE_string('snapshot_image_format', None,
                    'Snapshot image format (valid options are : '
                    'raw, qcow2, vmdk, vdi).'
                    'Defaults to same as source image')
flags.DEFINE_string('libvirt_vif_type', 'bridge',
                    'Type of VIF to create.')
flags.DEFINE_string('libvirt_vif_driver',
                    'nova.virt.libvirt.vif.LibvirtBridgeDriver',
                    'The libvirt VIF driver to configure the VIFs.')
flags.DEFINE_string('default_local_format',
                    None,
                    'The default format a local_volume will be formatted with '
                    'on creation.')
flags.DEFINE_bool('libvirt_use_virtio_for_bridges',
                  False,
                  'Use virtio for bridge interfaces')
flags.DEFINE_list('extra_node_capabilities',
                   [],
                   'Key/value list with extra caps of the compute node')
flags.DEFINE_string('dev_cgroups_path',
                    '/cgroup/devices/libvirt/lxc',
                    'LXC cgroups devices base path')

# Variables for tracking gpus available and gpus assigned
if FLAGS.connection_type == 'gpu':
    gpus_available = []
    gpus_assigned = {}
    vcpus_used = 0


def get_connection(read_only):
    # These are loaded late so that there's no need to install these
    # libraries when not using libvirt.
    # Cheetah is separate because the unit tests want to load Cheetah,
    # but not libvirt.
    global libvirt
    global libxml2
    if libvirt is None:
        libvirt = __import__('libvirt')
    if libxml2 is None:
        libxml2 = __import__('libxml2')
    _late_load_cheetah()
    return LibvirtConnection(read_only)


def _late_load_cheetah():
    global Template
    if Template is None:
        t = __import__('Cheetah.Template', globals(), locals(),
                       ['Template'], -1)
        Template = t.Template


def _get_eph_disk(ephemeral):
    return 'disk.eph' + str(ephemeral['num'])


class LibvirtConnection(driver.ComputeDriver):

    def __init__(self, read_only):
        super(LibvirtConnection, self).__init__()
        self.libvirt_uri = self.get_uri()

        self.libvirt_xml = open(FLAGS.libvirt_xml_template).read()
        self.cpuinfo_xml = open(FLAGS.cpuinfo_xml_template).read()
        self._wrapped_conn = None
        self.read_only = read_only

        fw_class = utils.import_class(FLAGS.firewall_driver)
        self.firewall_driver = fw_class(get_connection=self._get_connection)
        self.vif_driver = utils.import_object(FLAGS.libvirt_vif_driver)
        self._host_state = None
#        self.session = None

    @property
    def HostState(self):
        if not self._host_state:
            self._host_state = HostState(self.read_only)
        return self._host_state

    def init_host(self, host):
        if FLAGS.connection_type == 'gpu':
            global gpus_available
            gpus_available = range(FLAGS.xpus)

        # NOTE(nsokolov): moved instance restarting to ComputeManager
        pass

    def _get_connection(self):
        if not self._wrapped_conn or not self._test_connection():
            LOG.debug(_('Connecting to libvirt: %s'), self.libvirt_uri)
            self._wrapped_conn = self._connect(self.libvirt_uri,
                                               self.read_only)
        return self._wrapped_conn
    _conn = property(_get_connection)

    def _test_connection(self):
        try:
            self._wrapped_conn.getCapabilities()
            return True
        except libvirt.libvirtError as e:
            if e.get_error_code() == libvirt.VIR_ERR_SYSTEM_ERROR and \
               e.get_error_domain() == libvirt.VIR_FROM_REMOTE:
                LOG.debug(_('Connection to libvirt broke'))
                return False
            raise

    def get_uri(self):
        if FLAGS.libvirt_type == 'uml':
            uri = FLAGS.libvirt_uri or 'uml:///system'
        elif FLAGS.libvirt_type == 'xen':
            uri = FLAGS.libvirt_uri or 'xen:///'
        elif FLAGS.libvirt_type == 'lxc':
            uri = FLAGS.libvirt_uri or 'lxc:///'
        else:
            uri = FLAGS.libvirt_uri or 'qemu:///system'
        return uri

    def _connect(self, uri, read_only):
        auth = [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_NOECHOPROMPT],
                'root',
                None]

        if read_only:
            return libvirt.openReadOnly(uri)
        else:
            return libvirt.openAuth(uri, auth, 0)

    def list_instances(self):
        return [self._conn.lookupByID(x).name()
                for x in self._conn.listDomainsID()]

    def _map_to_instance_info(self, domain):
        """Gets info from a virsh domain object into an InstanceInfo"""

        # domain.info() returns a list of:
        #    state:       one of the state values (virDomainState)
        #    maxMemory:   the maximum memory used by the domain
        #    memory:      the current amount of memory used by the domain
        #    nbVirtCPU:   the number of virtual CPU
        #    puTime:      the time used by the domain in nanoseconds

        (state, _max_mem, _mem, _num_cpu, _cpu_time) = domain.info()
        name = domain.name()

        return driver.InstanceInfo(name, state)

    def list_instances_detail(self):
        infos = []
        for domain_id in self._conn.listDomainsID():
            domain = self._conn.lookupByID(domain_id)
            info = self._map_to_instance_info(domain)
            infos.append(info)
        return infos

    def allow_gpus(self, inst):
        dev_whitelist = os.path.join(FLAGS.dev_cgroups_path,
                                      inst['name'],
                                      'devices.allow')
        # Allow Nvidia Conroller
        perm = "c 195:255  rwm"
#        cmd = "sudo echo %s >> %s" % (perm, dev_whitelist)
        cmd = "echo %s | sudo tee -a %s" % (perm, dev_whitelist)
        msg = _("executing  %s") % cmd
        LOG.info(msg)
        subprocess.Popen(cmd, shell=True)
        for i in range(FLAGS.xpus):
            # Allow each gpu device
            perm = "c 195:%d  rwm" % i
#            cmd = "sudo echo %s >> %s" % (perm, dev_whitelist)
            cmd = "echo %s | sudo tee -a %s" % (perm, dev_whitelist)
            msg = _("executing  %s") % cmd
            LOG.info(msg)
            subprocess.Popen(cmd, shell=True)

    def assign_gpus(self, inst):
        """Assigns gpus to a specific instance"""
        global gpus_available
        global gpus_assigned
        global vcpus_used
        ctxt = nova_context.get_admin_context()
        gpus_in_meta = 0
        gpus_in_extra = 0
        env_file = os.path.join(FLAGS.instances_path,
                                inst['name'],
                                'rootfs/etc/environment')
        msg = _("instance_type_id is %d .") % inst.instance_type_id
        LOG.info(msg)
        print "(JP) %s" % msg
        instance_extra = db.instance_type_extra_specs_get( \
                                ctxt, inst.instance_type_id)
        msg = _("instance_extra is %s .") % instance_extra
        LOG.info(msg)
        msg = _("vcpus for this instance are %d .") % inst.vcpus
        LOG.info(msg)
        vcpus_used = vcpus_used + inst.vcpus
        if 'xpus' in inst['metadata']:
            gpus_in_meta = int(inst['metadata']['xpus'])
            msg = _("xpus in metadata asked, %d .") % gpus_in_meta
            LOG.info(msg)
        if 'xpus' in instance_extra:
            gpus_in_extra = int(instance_extra['xpus'])
            msg = _("xpus in instance_extra asked, %d .") % gpus_in_extra
            LOG.info(msg)

        if gpus_in_meta > gpus_in_extra:
            gpus_needed = gpus_in_meta
        else:
            gpus_needed = gpus_in_extra
        #added by JP for debug
        #gpus_needed = 1
        msg = _("GPUS asked, %d .") % gpus_needed
#        raise Exception(_("Test Error"))
        LOG.info(msg)
        print "(JP) %s" % msg
        # Allow gpu devices inside the container
        self.allow_gpus(inst)
        # Set environemnt
        gpus_assigned_list = []
#        self.destroy(inst, network_info)

        if gpus_needed > len(gpus_available):
            raise Exception(_("Overcommit Error"))
        for i in range(gpus_needed):
            gpus_assigned_list.append(gpus_available.pop())
        if gpus_needed:
            print 'GPUs are definitely needed.'
            gpus_assigned[inst['name']] = gpus_assigned_list
            gpus_visible = str(gpus_assigned_list).strip('[]')
            flag = "CUDA_VISIBLE_DEVICES=%s" % gpus_visible
            cmd = "echo %s | sudo tee -a %s" % (flag, env_file)
#            cmd = "echo %s >> %s" % (flag, env_file)
            msg = _("executing the command %s") % cmd
            LOG.info(msg)
            print "(JP) %s" % msg
            subprocess.Popen(cmd, shell=True)
#            utils.executeShell(cmd, run_as_root=True)

    def deassign_gpus(self, inst):
        """Assigns gpus to a specific instance"""
        global gpus_available
        global gpus_assigned
        global vcpus_used

        vcpus_used = vcpus_used - inst.vcpus
        if vcpus_used < 0:
            vcpus_used = 0
        #Check if already deassigned
        if inst['name'] in gpus_assigned:
            gpus_available.extend(gpus_assigned[inst['name']])
            del gpus_assigned[inst['name']]
        return

    def plug_vifs(self, instance, network_info):
        """Plugin VIFs into networks."""
        for (network, mapping) in network_info:
            self.vif_driver.plug(instance, network, mapping)

    def destroy(self, instance, network_info, cleanup=True):
        instance_name = instance['name']

        try:
            virt_dom = self._lookup_by_name(instance_name)
        except exception.NotFound:
            virt_dom = None

        # If the instance is already terminated, we're still happy
        # Otherwise, destroy it
        if virt_dom is not None:
            try:
                virt_dom.destroy()
            except libvirt.libvirtError as e:
                is_okay = False
                errcode = e.get_error_code()
                if errcode == libvirt.VIR_ERR_OPERATION_INVALID:
                    # If the instance if already shut off, we get this:
                    # Code=55 Error=Requested operation is not valid:
                    # domain is not running
                    (state, _max_mem, _mem, _cpus, _t) = virt_dom.info()
                    if state == power_state.SHUTOFF:
                        is_okay = True

                if not is_okay:
                    LOG.warning(_("Error from libvirt during destroy of "
                                  "%(instance_name)s. Code=%(errcode)s "
                                  "Error=%(e)s") %
                                locals())
                    raise

            try:
                # NOTE(justinsb): We remove the domain definition. We probably
                # would do better to keep it if cleanup=False (e.g. volumes?)
                # (e.g. #2 - not losing machines on failure)
                virt_dom.undefine()
            except libvirt.libvirtError as e:
                errcode = e.get_error_code()
                LOG.warning(_("Error from libvirt during undefine of "
                              "%(instance_name)s. Code=%(errcode)s "
                              "Error=%(e)s") %
                            locals())
                raise

            for (network, mapping) in network_info:
                self.vif_driver.unplug(instance, network, mapping)

        def _wait_for_destroy():
            """Called at an interval until the VM is gone."""
            instance_name = instance['name']

            try:
                state = self.get_info(instance_name)['state']
            except exception.NotFound:
                msg = _("Instance %s destroyed successfully.") % instance_name
                LOG.info(msg)
                raise utils.LoopingCallDone

        timer = utils.LoopingCall(_wait_for_destroy)
        timer.start(interval=0.5, now=True)

        self.firewall_driver.unfilter_instance(instance,
                                               network_info=network_info)

        if cleanup:
            self._cleanup(instance)

        return True

    def _cleanup(self, instance):
        target = os.path.join(FLAGS.instances_path, instance['name'])
        instance_name = instance['name']
        LOG.info(_('instance %(instance_name)s: deleting instance files'
                ' %(target)s') % locals())
        if FLAGS.libvirt_type == 'lxc':
            try:
                disk.destroy_container(target, instance, \
                    nbd=FLAGS.use_cow_images)
            except:
                pass
        if FLAGS.connection_type == 'gpu':
            self.deassign_gpus(instance)
        if os.path.exists(target):
            try:
                shutil.rmtree(target)
            except:
                pass

    @exception.wrap_exception()
    def attach_volume(self, instance_name, device_path, mountpoint):
        virt_dom = self._lookup_by_name(instance_name)
# ISI
        if FLAGS.libvirt_type == 'lxc':
            LOG.info(_('attach_volume: path(%s)') % device_path)
            # get id of the virt_dom
            pid = virt_dom.ID()
            spid = str(pid)
            LOG.info(_('attach_volume: pid(%s)') % spid)

            # get PID of the init process
            ps_command = subprocess.Popen("ps -o pid --ppid %s --noheaders" % spid, shell=True, stdout=subprocess.PIPE)
            init_pid = ps_command.stdout.read()
            init_pid = int(init_pid)
            init_pid = str(init_pid)
            retcode = ps_command.wait()
            assert retcode == 0, "ps command returned %d" % retcode

            LOG.info(_('attach_volume: init_pid(%s)') % init_pid)
            # get major, minor number of the device
            s = os.stat(device_path)
            major_num = os.major(s.st_rdev)
            minor_num = os.minor(s.st_rdev)
            LOG.info(_('attach_volume: path(%s)') % device_path)
            LOG.info(_('attach_volume: major_num(%(major_num)d) minor_num(%(minor_num)d)') % locals())

            # allow the device
            dev_whitelist = os.path.join(FLAGS.dev_cgroups_path,
                                      instance_name,
                                      'devices.allow')
            # Allow the disk
            perm = "b %d:%d rwm" % (major_num, minor_num)
            cmd = "echo %s | sudo tee -a %s" % (perm, dev_whitelist)
            LOG.info(_('attach_volume: cmd(%s)') % cmd)
            subprocess.Popen(cmd, shell=True)

            # pass the numbers to the LXC instance
            # run lxc-attach:
            # sudo lxc-attach -n pid -- mknod -m 777
            #                 <mountpoint> b <major #> <minor #> 
            cmd_lxc = 'sudo lxc-attach -n %s -- ' % init_pid
            # check if 'mountpoint' already exists
            #cmd = '/bin/ls %s' % mountpoint
            #if subprocess.call(cmd, shell=True) == 0: # not new
            #    postfix = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', \
            #               'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
            #    for t in range (len(postfix)):
            #        n_mountpoint = mountpoint + postfix[t]
            #        cmd = '/bin/ls %s' % n_mountpoint
            #        if subprocess.call(cmd, shell=True) <> 0: # new
            #            break;

            LOG.info(_('attach_volume: mountpoint(%s)') % mountpoint)
            dev_key = init_pid + mountpoint
            LOG.info(_('attach_volume: dev_key(%s)') % dev_key)
            if dev_key in lxc_mounts:
                LOG.info(_('attach_volume: dev_key(%s) is already used') % dev_key)
                raise Exception(_('the same mount point(%s) is already used.') % mountpoint)

            # create device(s) for mount
            cmd = '/bin/mknod -m 777 %s b %d %d '\
                 % (mountpoint, major_num, minor_num)
            cmd = cmd_lxc + cmd
            LOG.info(_('attach_volume: cmd (%s)') % cmd)
            subprocess.call(cmd, shell=True)

            # Allow the disk
            #perm = "b %d:%d rwm" % (major_num, minor_num)
            #cmd = "echo %s | sudo tee -a %s" % (perm, dev_whitelist)
            #LOG.info(_('attach_volume: cmd(%s)') % cmd)
            #subprocess.call(cmd, shell=True)

            #cmd = '/bin/mknod -m 777 %s b %d %d '\
            #     % (mountpoint, major_num, minor_num)
            #cmd = cmd_lxc + cmd
            #LOG.info(_('attach_volume: cmd (%s)') % cmd)
            #subprocess.call(cmd, shell=True)
           

            # create a directory for mount
            found = 0
            for n in range (0, 100):
                dir_name = '/euca-volume' + str(n)
                cmd1 = cmd_lxc + ' /bin/ls ' + dir_name
                LOG.info(_('attach_volume: cmd (%s)') % cmd1)
                p = subprocess.Popen(cmd1, shell=True,  \
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE) 
                x = p.communicate()
                LOG.info(_('attach_volume: return (%(x[0])s, %(x[1])s)') % locals())
                #if len(x[1]) > 5: # new  "No such file exists..."
                s = x[1].lower()
                if (len(s) > 0 and s.find('no such') >= 0) : # new  "No such file exists..."
                    cmd2 = cmd_lxc + ' /bin/mkdir ' + dir_name
                    LOG.info(_('attach_volume: cmd (%s)') % cmd2)
                    subprocess.call(cmd2, shell=True)
                    found = 1
                    break;
            if found == 0:
                cmd = '/bin/rm %s '\
                     % (mountpoint)
                cmd = cmd_lxc + cmd
                LOG.info(_('attach_volume: cmd (%s)') % cmd)
                subprocess.call(cmd, shell=True)
                raise Exception(_('cannot find mounting directories'))

            lxc_mounts[dev_key] = dir_name 
            # mount
            cmd1 = cmd_lxc + ' /bin/mount ' + mountpoint + ' ' + dir_name
            LOG.info(_('attach_volume: cmd (%s)') % cmd1)
            subprocess.call(cmd1, shell=True)

            cmd1 = cmd_lxc + " /bin/chmod 'og+w' " + ' ' + dir_name
            LOG.info(_('attach_volume: cmd (%s)') % cmd1)
            subprocess.call(cmd1, shell=True)

            return
# !ISI
        mount_device = mountpoint.rpartition("/")[2]
        (type, protocol, name) = \
            self._get_volume_device_info(device_path)
        if type == 'block':
            xml = """<disk type='block'>
                         <driver name='qemu' type='raw'/>
                         <source dev='%s'/>
                         <target dev='%s' bus='virtio'/>
                     </disk>""" % (device_path, mount_device)
        elif type == 'network':
            xml = """<disk type='network'>
                         <driver name='qemu' type='raw'/>
                         <source protocol='%s' name='%s'/>
                         <target dev='%s' bus='virtio'/>
                     </disk>""" % (protocol, name, mount_device)
        virt_dom.attachDevice(xml)

    def _get_disk_xml(self, xml, device):
        """Returns the xml for the disk mounted at device"""
        try:
            doc = libxml2.parseDoc(xml)
        except Exception:
            return None
        ctx = doc.xpathNewContext()
        try:
            ret = ctx.xpathEval('/domain/devices/disk')
            for node in ret:
                for child in node.children:
                    if child.name == 'target':
                        if child.prop('dev') == device:
                            return str(node)
        finally:
            if ctx is not None:
                ctx.xpathFreeContext()
            if doc is not None:
                doc.freeDoc()

    @exception.wrap_exception()
    def detach_volume(self, instance_name, mountpoint):
        virt_dom = self._lookup_by_name(instance_name)
# ISI
        if FLAGS.libvirt_type == 'lxc':
            #LOG.info(_('detach_volume: path(%s): do nothing'))
            #return
            # get id of the virt_dom
            pid = virt_dom.ID()
            spid = str(pid)
            LOG.info(_('detach_volume: pid(%s)') % spid)

            # get PID of the init process
            ps_command = subprocess.Popen("ps -o pid --ppid %s --noheaders" % spid, shell=True, stdout=subprocess.PIPE)
            init_pid = ps_command.stdout.read()
            init_pid = int(init_pid)
            init_pid = str(init_pid)
            retcode = ps_command.wait()
            assert retcode == 0, "ps command returned %d" % retcode

            dev_key = init_pid + mountpoint
            if dev_key not in lxc_mounts:
                raise Exception(_('no such process(%(init_pid)s) or mount point(%(mountpoint)s)') % locals())
            dir_name = lxc_mounts[dev_key]

            LOG.info(_('detach_volume: init_pid(%s)') % init_pid)
            cmd_lxc = 'sudo lxc-attach -n %s -- ' % str(init_pid)
            cmd1 = cmd_lxc + ' /bin/umount ' + dir_name
            LOG.info(_('detach_volume: cmd(%s)') % cmd1)
            subprocess.call(cmd1, shell=True)
            cmd1 = cmd_lxc + ' /bin/rmdir  ' + dir_name
            LOG.info(_('detach_volume: cmd(%s)') % cmd1)
            subprocess.call(cmd1, shell=True)

            del lxc_mounts[dev_key]  # delete dictionary entry

            cmd1 = cmd_lxc + ' /bin/rm ' + mountpoint
            LOG.info(_('detach_volume: cmd(%s)') % cmd1)
            subprocess.call(cmd1, shell=True)
            return
# !ISI
        
        mount_device = mountpoint.rpartition("/")[2]
        xml = self._get_disk_xml(virt_dom.XMLDesc(0), mount_device)
        if not xml:
            raise exception.DiskNotFound(location=mount_device)
        virt_dom.detachDevice(xml)

    @exception.wrap_exception()
    def snapshot(self, context, instance, image_href):
        """Create snapshot from a running VM instance.

        This command only works with qemu 0.14+
        """
        virt_dom = self._lookup_by_name(instance['name'])<commit_msg>Fixed pep8 errors<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright (c) 2010 Citrix Systems, Inc.
# Copyright (c) 2011 Piston Cloud Computing, Inc
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
A connection to a hypervisor through libvirt.

Supports KVM, LXC, QEMU, UML, and XEN.

**Related Flags**

:libvirt_type:  Libvirt domain type.  Can be kvm, qemu, uml, xen
                (default: kvm).
:libvirt_uri:  Override for the default libvirt URI (depends on libvirt_type).
:libvirt_xml_template:  Libvirt XML Template.
:rescue_image_id:  Rescue ami image (None = original image).
:rescue_kernel_id:  Rescue aki image (None = original image).
:rescue_ramdisk_id:  Rescue ari image (None = original image).
:injected_network_template:  Template file for injected network
:allow_same_net_traffic:  Whether to allow in project network traffic

"""

import hashlib
import functools
import multiprocessing
import netaddr
import os
import random
import re
import shutil
import sys
import tempfile
import time
import uuid
import subprocess
from xml.dom import minidom
from xml.etree import ElementTree

from eventlet import greenthread
from eventlet import tpool

from nova import block_device
from nova import context as nova_context
from nova import db
from nova import exception
from nova import flags
import nova.image
from nova import log as logging
from nova import utils
from nova import vnc
from nova.auth import manager
from nova.compute import instance_types
from nova.compute import power_state
from nova.compute import vm_states
from nova.virt import disk
from nova.virt import driver
from nova.virt import images
from nova.virt.libvirt import netutils


libvirt = None
libxml2 = None
Template = None

LOG = logging.getLogger('nova.virt.libvirt_conn')

#ISI
lxc_mounts = {}
#!ISI

FLAGS = flags.FLAGS
flags.DECLARE('live_migration_retry_count', 'nova.compute.manager')
# TODO(vish): These flags should probably go into a shared location
flags.DEFINE_string('rescue_image_id', None, 'Rescue ami image')
flags.DEFINE_string('rescue_kernel_id', None, 'Rescue aki image')
flags.DEFINE_string('rescue_ramdisk_id', None, 'Rescue ari image')
flags.DEFINE_string('libvirt_xml_template',
                    utils.abspath('virt/libvirt.xml.template'),
                    'Libvirt XML Template')
flags.DEFINE_string('libvirt_type',
                    'kvm',
                    'Libvirt domain type (valid options are: '
                    'kvm, lxc, qemu, uml, xen)')
flags.DEFINE_string('libvirt_uri',
                    '',
                    'Override the default libvirt URI (which is dependent'
                    ' on libvirt_type)')
flags.DEFINE_bool('allow_same_net_traffic',
                  True,
                  'Whether to allow network traffic from same network')
flags.DEFINE_bool('use_cow_images',
                  True,
                  'Whether to use cow images')
flags.DEFINE_string('ajaxterm_portrange',
                    '10000-12000',
                    'Range of ports that ajaxterm should randomly try to bind')
flags.DEFINE_string('firewall_driver',
                    'nova.virt.libvirt.firewall.IptablesFirewallDriver',
                    'Firewall driver (defaults to iptables)')
flags.DEFINE_string('cpuinfo_xml_template',
                    utils.abspath('virt/cpuinfo.xml.template'),
                    'CpuInfo XML Template (Used only live migration now)')
flags.DEFINE_string('live_migration_uri',
                    "qemu+tcp://%s/system",
                    'Define protocol used by live_migration feature')
flags.DEFINE_string('live_migration_flag',
                    "VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER",
                    'Define live migration behavior.')
flags.DEFINE_string('block_migration_flag',
                    "VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER, "
                    "VIR_MIGRATE_NON_SHARED_INC",
                    'Define block migration behavior.')
flags.DEFINE_integer('live_migration_bandwidth', 0,
                    'Define live migration behavior')
flags.DEFINE_string('snapshot_image_format', None,
                    'Snapshot image format (valid options are : '
                    'raw, qcow2, vmdk, vdi).'
                    'Defaults to same as source image')
flags.DEFINE_string('libvirt_vif_type', 'bridge',
                    'Type of VIF to create.')
flags.DEFINE_string('libvirt_vif_driver',
                    'nova.virt.libvirt.vif.LibvirtBridgeDriver',
                    'The libvirt VIF driver to configure the VIFs.')
flags.DEFINE_string('default_local_format',
                    None,
                    'The default format a local_volume will be formatted with '
                    'on creation.')
flags.DEFINE_bool('libvirt_use_virtio_for_bridges',
                  False,
                  'Use virtio for bridge interfaces')
flags.DEFINE_list('extra_node_capabilities',
                   [],
                   'Key/value list with extra caps of the compute node')
flags.DEFINE_string('dev_cgroups_path',
                    '/cgroup/devices/libvirt/lxc',
                    'LXC cgroups devices base path')

# Variables for tracking gpus available and gpus assigned
if FLAGS.connection_type == 'gpu':
    gpus_available = []
    gpus_assigned = {}
    vcpus_used = 0


def get_connection(read_only):
    # These are loaded late so that there's no need to install these
    # libraries when not using libvirt.
    # Cheetah is separate because the unit tests want to load Cheetah,
    # but not libvirt.
    global libvirt
    global libxml2
    if libvirt is None:
        libvirt = __import__('libvirt')
    if libxml2 is None:
        libxml2 = __import__('libxml2')
    _late_load_cheetah()
    return LibvirtConnection(read_only)


def _late_load_cheetah():
    global Template
    if Template is None:
        t = __import__('Cheetah.Template', globals(), locals(),
                       ['Template'], -1)
        Template = t.Template


def _get_eph_disk(ephemeral):
    return 'disk.eph' + str(ephemeral['num'])


class LibvirtConnection(driver.ComputeDriver):

    def __init__(self, read_only):
        super(LibvirtConnection, self).__init__()
        self.libvirt_uri = self.get_uri()

        self.libvirt_xml = open(FLAGS.libvirt_xml_template).read()
        self.cpuinfo_xml = open(FLAGS.cpuinfo_xml_template).read()
        self._wrapped_conn = None
        self.read_only = read_only

        fw_class = utils.import_class(FLAGS.firewall_driver)
        self.firewall_driver = fw_class(get_connection=self._get_connection)
        self.vif_driver = utils.import_object(FLAGS.libvirt_vif_driver)
        self._host_state = None
#        self.session = None

    @property
    def HostState(self):
        if not self._host_state:
            self._host_state = HostState(self.read_only)
        return self._host_state

    def init_host(self, host):
        if FLAGS.connection_type == 'gpu':
            global gpus_available
            gpus_available = range(FLAGS.xpus)

        # NOTE(nsokolov): moved instance restarting to ComputeManager
        pass

    def _get_connection(self):
        if not self._wrapped_conn or not self._test_connection():
            LOG.debug(_('Connecting to libvirt: %s'), self.libvirt_uri)
            self._wrapped_conn = self._connect(self.libvirt_uri,
                                               self.read_only)
        return self._wrapped_conn
    _conn = property(_get_connection)

    def _test_connection(self):
        try:
            self._wrapped_conn.getCapabilities()
            return True
        except libvirt.libvirtError as e:
            if e.get_error_code() == libvirt.VIR_ERR_SYSTEM_ERROR and \
               e.get_error_domain() == libvirt.VIR_FROM_REMOTE:
                LOG.debug(_('Connection to libvirt broke'))
                return False
            raise

    def get_uri(self):
        if FLAGS.libvirt_type == 'uml':
            uri = FLAGS.libvirt_uri or 'uml:///system'
        elif FLAGS.libvirt_type == 'xen':
            uri = FLAGS.libvirt_uri or 'xen:///'
        elif FLAGS.libvirt_type == 'lxc':
            uri = FLAGS.libvirt_uri or 'lxc:///'
        else:
            uri = FLAGS.libvirt_uri or 'qemu:///system'
        return uri

    def _connect(self, uri, read_only):
        auth = [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_NOECHOPROMPT],
                'root',
                None]

        if read_only:
            return libvirt.openReadOnly(uri)
        else:
            return libvirt.openAuth(uri, auth, 0)

    def list_instances(self):
        return [self._conn.lookupByID(x).name()
                for x in self._conn.listDomainsID()]

    def _map_to_instance_info(self, domain):
        """Gets info from a virsh domain object into an InstanceInfo"""

        # domain.info() returns a list of:
        #    state:       one of the state values (virDomainState)
        #    maxMemory:   the maximum memory used by the domain
        #    memory:      the current amount of memory used by the domain
        #    nbVirtCPU:   the number of virtual CPU
        #    puTime:      the time used by the domain in nanoseconds

        (state, _max_mem, _mem, _num_cpu, _cpu_time) = domain.info()
        name = domain.name()

        return driver.InstanceInfo(name, state)

    def list_instances_detail(self):
        infos = []
        for domain_id in self._conn.listDomainsID():
            domain = self._conn.lookupByID(domain_id)
            info = self._map_to_instance_info(domain)
            infos.append(info)
        return infos

    def allow_gpus(self, inst):
        dev_whitelist = os.path.join(FLAGS.dev_cgroups_path,
                                      inst['name'],
                                      'devices.allow')
        # Allow Nvidia Conroller
        perm = "c 195:255  rwm"
#        cmd = "sudo echo %s >> %s" % (perm, dev_whitelist)
        cmd = "echo %s | sudo tee -a %s" % (perm, dev_whitelist)
        msg = _("executing  %s") % cmd
        LOG.info(msg)
        subprocess.Popen(cmd, shell=True)
        for i in range(FLAGS.xpus):
            # Allow each gpu device
            perm = "c 195:%d  rwm" % i
#            cmd = "sudo echo %s >> %s" % (perm, dev_whitelist)
            cmd = "echo %s | sudo tee -a %s" % (perm, dev_whitelist)
            msg = _("executing  %s") % cmd
            LOG.info(msg)
            subprocess.Popen(cmd, shell=True)

    def assign_gpus(self, inst):
        """Assigns gpus to a specific instance"""
        global gpus_available
        global gpus_assigned
        global vcpus_used
        ctxt = nova_context.get_admin_context()
        gpus_in_meta = 0
        gpus_in_extra = 0
        env_file = os.path.join(FLAGS.instances_path,
                                inst['name'],
                                'rootfs/etc/environment')
        msg = _("instance_type_id is %d .") % inst.instance_type_id
        LOG.info(msg)
        print "(JP) %s" % msg
        instance_extra = db.instance_type_extra_specs_get( \
                                ctxt, inst.instance_type_id)
        msg = _("instance_extra is %s .") % instance_extra
        LOG.info(msg)
        msg = _("vcpus for this instance are %d .") % inst.vcpus
        LOG.info(msg)
        vcpus_used = vcpus_used + inst.vcpus
        if 'xpus' in inst['metadata']:
            gpus_in_meta = int(inst['metadata']['xpus'])
            msg = _("xpus in metadata asked, %d .") % gpus_in_meta
            LOG.info(msg)
        if 'xpus' in instance_extra:
            gpus_in_extra = int(instance_extra['xpus'])
            msg = _("xpus in instance_extra asked, %d .") % gpus_in_extra
            LOG.info(msg)

        if gpus_in_meta > gpus_in_extra:
            gpus_needed = gpus_in_meta
        else:
            gpus_needed = gpus_in_extra
        #added by JP for debug
        #gpus_needed = 1
        msg = _("GPUS asked, %d .") % gpus_needed
#        raise Exception(_("Test Error"))
        LOG.info(msg)
        print "(JP) %s" % msg
        # Allow gpu devices inside the container
        self.allow_gpus(inst)
        # Set environemnt
        gpus_assigned_list = []
#        self.destroy(inst, network_info)

        if gpus_needed > len(gpus_available):
            raise Exception(_("Overcommit Error"))
        for i in range(gpus_needed):
            gpus_assigned_list.append(gpus_available.pop())
        if gpus_needed:
            print 'GPUs are definitely needed.'
            gpus_assigned[inst['name']] = gpus_assigned_list
            gpus_visible = str(gpus_assigned_list).strip('[]')
            flag = "CUDA_VISIBLE_DEVICES=%s" % gpus_visible
            cmd = "echo %s | sudo tee -a %s" % (flag, env_file)
#            cmd = "echo %s >> %s" % (flag, env_file)
            msg = _("executing the command %s") % cmd
            LOG.info(msg)
            print "(JP) %s" % msg
            subprocess.Popen(cmd, shell=True)
#            utils.executeShell(cmd, run_as_root=True)

    def deassign_gpus(self, inst):
        """Assigns gpus to a specific instance"""
        global gpus_available
        global gpus_assigned
        global vcpus_used

        vcpus_used = vcpus_used - inst.vcpus
        if vcpus_used < 0:
            vcpus_used = 0
        #Check if already deassigned
        if inst['name'] in gpus_assigned:
            gpus_available.extend(gpus_assigned[inst['name']])
            del gpus_assigned[inst['name']]
        return

    def plug_vifs(self, instance, network_info):
        """Plugin VIFs into networks."""
        for (network, mapping) in network_info:
            self.vif_driver.plug(instance, network, mapping)

    def destroy(self, instance, network_info, cleanup=True):
        instance_name = instance['name']

        try:
            virt_dom = self._lookup_by_name(instance_name)
        except exception.NotFound:
            virt_dom = None

        # If the instance is already terminated, we're still happy
        # Otherwise, destroy it
        if virt_dom is not None:
            try:
                virt_dom.destroy()
            except libvirt.libvirtError as e:
                is_okay = False
                errcode = e.get_error_code()
                if errcode == libvirt.VIR_ERR_OPERATION_INVALID:
                    # If the instance if already shut off, we get this:
                    # Code=55 Error=Requested operation is not valid:
                    # domain is not running
                    (state, _max_mem, _mem, _cpus, _t) = virt_dom.info()
                    if state == power_state.SHUTOFF:
                        is_okay = True

                if not is_okay:
                    LOG.warning(_("Error from libvirt during destroy of "
                                  "%(instance_name)s. Code=%(errcode)s "
                                  "Error=%(e)s") %
                                locals())
                    raise

            try:
                # NOTE(justinsb): We remove the domain definition. We probably
                # would do better to keep it if cleanup=False (e.g. volumes?)
                # (e.g. #2 - not losing machines on failure)
                virt_dom.undefine()
            except libvirt.libvirtError as e:
                errcode = e.get_error_code()
                LOG.warning(_("Error from libvirt during undefine of "
                              "%(instance_name)s. Code=%(errcode)s "
                              "Error=%(e)s") %
                            locals())
                raise

            for (network, mapping) in network_info:
                self.vif_driver.unplug(instance, network, mapping)

        def _wait_for_destroy():
            """Called at an interval until the VM is gone."""
            instance_name = instance['name']

            try:
                state = self.get_info(instance_name)['state']
            except exception.NotFound:
                msg = _("Instance %s destroyed successfully.") % instance_name
                LOG.info(msg)
                raise utils.LoopingCallDone

        timer = utils.LoopingCall(_wait_for_destroy)
        timer.start(interval=0.5, now=True)

        self.firewall_driver.unfilter_instance(instance,
                                               network_info=network_info)

        if cleanup:
            self._cleanup(instance)

        return True

    def _cleanup(self, instance):
        target = os.path.join(FLAGS.instances_path, instance['name'])
        instance_name = instance['name']
        LOG.info(_('instance %(instance_name)s: deleting instance files'
                ' %(target)s') % locals())
        if FLAGS.libvirt_type == 'lxc':
            try:
                disk.destroy_container(target, instance, \
                    nbd=FLAGS.use_cow_images)
            except:
                pass
        if FLAGS.connection_type == 'gpu':
            self.deassign_gpus(instance)
        if os.path.exists(target):
            try:
                shutil.rmtree(target)
            except:
                pass

    @exception.wrap_exception()
    def attach_volume(self, instance_name, device_path, mountpoint):
        virt_dom = self._lookup_by_name(instance_name)
# ISI
        if FLAGS.libvirt_type == 'lxc':
            LOG.info(_('attach_volume: path(%s)') % device_path)
            # get id of the virt_dom
            pid = virt_dom.ID()
            spid = str(pid)
            LOG.info(_('attach_volume: pid(%s)') % spid)

            # get PID of the init process
            ps_command = subprocess.Popen("ps -o pid --ppid %s --noheaders" % \
                                    spid, shell=True, stdout=subprocess.PIPE)
            init_pid = ps_command.stdout.read()
            init_pid = int(init_pid)
            init_pid = str(init_pid)
            retcode = ps_command.wait()
            assert retcode == 0, "ps command returned %d" % retcode

            LOG.info(_('attach_volume: init_pid(%s)') % init_pid)
            # get major, minor number of the device
            s = os.stat(device_path)
            major_num = os.major(s.st_rdev)
            minor_num = os.minor(s.st_rdev)
            LOG.info(_('attach_volume: path(%s)') % device_path)
            LOG.info(_('attach_volume: major_num(%(major_num)d) ' \
                       'minor_num(%(minor_num)d)') % locals())

            # allow the device
            dev_whitelist = os.path.join(FLAGS.dev_cgroups_path,
                                      instance_name,
                                      'devices.allow')
            # Allow the disk
            perm = "b %d:%d rwm" % (major_num, minor_num)
            cmd = "echo %s | sudo tee -a %s" % (perm, dev_whitelist)
            LOG.info(_('attach_volume: cmd(%s)') % cmd)
            subprocess.Popen(cmd, shell=True)

            # pass the numbers to the LXC instance
            # run lxc-attach:
            # sudo lxc-attach -n pid -- mknod -m 777
            #                 <mountpoint> b <major #> <minor #>
            cmd_lxc = 'sudo lxc-attach -n %s -- ' % init_pid
            # check if 'mountpoint' already exists
            #cmd = '/bin/ls %s' % mountpoint
            #if subprocess.call(cmd, shell=True) == 0: # not new
            #    postfix = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', \
            #               'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', \
            #               's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
            #    for t in range (len(postfix)):
            #        n_mountpoint = mountpoint + postfix[t]
            #        cmd = '/bin/ls %s' % n_mountpoint
            #        if subprocess.call(cmd, shell=True) <> 0: # new
            #            break;

            LOG.info(_('attach_volume: mountpoint(%s)') % mountpoint)
            dev_key = init_pid + mountpoint
            LOG.info(_('attach_volume: dev_key(%s)') % dev_key)
            if dev_key in lxc_mounts:
                LOG.info(_('attach_volume: dev_key(%s) is already used') \
                            % dev_key)
                raise Exception(_('the same mount point(%s) is already used.')\
                            % mountpoint)

            # create device(s) for mount
            cmd = '/bin/mknod -m 777 %s b %d %d '\
                 % (mountpoint, major_num, minor_num)
            cmd = cmd_lxc + cmd
            LOG.info(_('attach_volume: cmd (%s)') % cmd)
            subprocess.call(cmd, shell=True)

            # Allow the disk
            #perm = "b %d:%d rwm" % (major_num, minor_num)
            #cmd = "echo %s | sudo tee -a %s" % (perm, dev_whitelist)
            #LOG.info(_('attach_volume: cmd(%s)') % cmd)
            #subprocess.call(cmd, shell=True)

            #cmd = '/bin/mknod -m 777 %s b %d %d ' \
            #     % (mountpoint, major_num, minor_num)
            #cmd = cmd_lxc + cmd
            #LOG.info(_('attach_volume: cmd (%s)') % cmd)
            #subprocess.call(cmd, shell=True)

            # create a directory for mount
            found = 0
            for n in range(0, 100):
                dir_name = '/euca-volume' + str(n)
                cmd1 = cmd_lxc + ' /bin/ls ' + dir_name
                LOG.info(_('attach_volume: cmd (%s)') % cmd1)
                p = subprocess.Popen(cmd1, shell=True,  \
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                x = p.communicate()
                LOG.info(_('attach_volume: return (%(x[0])s, %(x[1])s)') \
                            % locals())
                #if len(x[1]) > 5: # new  "No such file exists..."
                s = x[1].lower()
                if (len(s) > 0 and s.find('no such') >= 0):
                # new  "No such file exists..."
                    cmd2 = cmd_lxc + ' /bin/mkdir ' + dir_name
                    LOG.info(_('attach_volume: cmd (%s)') % cmd2)
                    subprocess.call(cmd2, shell=True)
                    found = 1
                    break
            if found == 0:
                cmd = '/bin/rm %s '\
                     % (mountpoint)
                cmd = cmd_lxc + cmd
                LOG.info(_('attach_volume: cmd (%s)') % cmd)
                subprocess.call(cmd, shell=True)
                raise Exception(_('cannot find mounting directories'))

            lxc_mounts[dev_key] = dir_name
            # mount
            cmd1 = cmd_lxc + ' /bin/mount ' + mountpoint + ' ' + dir_name
            LOG.info(_('attach_volume: cmd (%s)') % cmd1)
            subprocess.call(cmd1, shell=True)

            cmd1 = cmd_lxc + " /bin/chmod 'og+w' " + ' ' + dir_name
            LOG.info(_('attach_volume: cmd (%s)') % cmd1)
            subprocess.call(cmd1, shell=True)

            return
# !ISI
        mount_device = mountpoint.rpartition("/")[2]
        (type, protocol, name) = \
            self._get_volume_device_info(device_path)
        if type == 'block':
            xml = """<disk type='block'>
                         <driver name='qemu' type='raw'/>
                         <source dev='%s'/>
                         <target dev='%s' bus='virtio'/>
                     </disk>""" % (device_path, mount_device)
        elif type == 'network':
            xml = """<disk type='network'>
                         <driver name='qemu' type='raw'/>
                         <source protocol='%s' name='%s'/>
                         <target dev='%s' bus='virtio'/>
                     </disk>""" % (protocol, name, mount_device)
        virt_dom.attachDevice(xml)

    def _get_disk_xml(self, xml, device):
        """Returns the xml for the disk mounted at device"""
        try:
            doc = libxml2.parseDoc(xml)
        except Exception:
            return None
        ctx = doc.xpathNewContext()
        try:
            ret = ctx.xpathEval('/domain/devices/disk')
            for node in ret:
                for child in node.children:
                    if child.name == 'target':
                        if child.prop('dev') == device:
                            return str(node)
        finally:
            if ctx is not None:
                ctx.xpathFreeContext()
            if doc is not None:
                doc.freeDoc()

    @exception.wrap_exception()
    def detach_volume(self, instance_name, mountpoint):
        virt_dom = self._lookup_by_name(instance_name)
# ISI
        if FLAGS.libvirt_type == 'lxc':
            #LOG.info(_('detach_volume: path(%s): do nothing'))
            #return
            # get id of the virt_dom
            pid = virt_dom.ID()
            spid = str(pid)
            LOG.info(_('detach_volume: pid(%s)') % spid)

            # get PID of the init process
            ps_command = subprocess.Popen("ps -o pid --ppid %s --noheaders" \
                                  % spid, shell=True, stdout=subprocess.PIPE)
            init_pid = ps_command.stdout.read()
            init_pid = int(init_pid)
            init_pid = str(init_pid)
            retcode = ps_command.wait()
            assert retcode == 0, "ps command returned %d" % retcode

            dev_key = init_pid + mountpoint
            if dev_key not in lxc_mounts:
                raise Exception(_('no such process(%(init_pid)s) or ' \
                      'mount point(%(mountpoint)s)') % locals())
            dir_name = lxc_mounts[dev_key]

            LOG.info(_('detach_volume: init_pid(%s)') % init_pid)
            cmd_lxc = 'sudo lxc-attach -n %s -- ' % str(init_pid)
            cmd1 = cmd_lxc + ' /bin/umount ' + dir_name
            LOG.info(_('detach_volume: cmd(%s)') % cmd1)
            subprocess.call(cmd1, shell=True)
            cmd1 = cmd_lxc + ' /bin/rmdir  ' + dir_name
            LOG.info(_('detach_volume: cmd(%s)') % cmd1)
            subprocess.call(cmd1, shell=True)

            del lxc_mounts[dev_key]  # delete dictionary entry

            cmd1 = cmd_lxc + ' /bin/rm ' + mountpoint
            LOG.info(_('detach_volume: cmd(%s)') % cmd1)
            subprocess.call(cmd1, shell=True)
            return
# !ISI

        mount_device = mountpoint.rpartition("/")[2]
        xml = self._get_disk_xml(virt_dom.XMLDesc(0), mount_device)
        if not xml:
            raise exception.DiskNotFound(location=mount_device)
        virt_dom.detachDevice(xml)

    @exception.wrap_exception()
    def snapshot(self, context, instance, image_href):
        """Create snapshot from a running VM instance.

        This command only works with qemu 0.14+
        """
        virt_dom = self._lookup_by_name(instance['name'])
{'fraction_non_alphanumeric': 0.10426225883133396, 'fraction_numerical': 0.004256730609762839, 'mean_word_length': 1.8634444913465598, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 56}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '644789', 'n_tokens_mistral': 16507, 'n_tokens_neox': 15819, 'n_words': 4050}