<commit_before><commit_msg>Add a script which can import VMs created by 'boxgrinder'<commit_after>#!/usr/bin/env python
#
# Copyright (C) Citrix Inc
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# Import appliances generated by boxgrinder into XenServer/XCP

import os, sys, time, socket, traceback, syslog

log_f = os.fdopen(os.dup(sys.stdout.fileno()), "aw")
pid = None
use_syslog = False

def reopenlog(log_file):
    global log_f
    if log_f:
        log_f.close()
    if log_file and log_file <> "stdout:":
        log_f = open(log_file, "aw")
    elif log_file and log_file == "stdout:":
        log_f = os.fdopen(os.dup(sys.stdout.fileno()), "aw")

def log(txt):
    global log_f, pid, use_syslog
    if use_syslog:
        syslog.syslog(txt)
        return
    if not pid:
        pid = os.getpid()
    t = time.strftime("%Y%m%dT%H:%M:%SZ", time.gmtime())
    print >>log_f, "%s [%d] %s" % (t, pid, txt)
    log_f.flush()

# For reference, here's what the boxgrinder default output XML looks like
# Is there a definition somewhere?
example = """
<image>
  <name>centos-base</name>
  <domain>
    <boot type='hvm'>
      <guest>
        <arch>x86_64</arch>
      </guest>
      <os>
        <loader dev='hd'/>
      </os>
      <drive disk='centos-base-sda.raw' target='hda'/>
    </boot>
    <devices>
      <vcpu>1</vcpu>
      <memory>262144</memory>
      <interface/>
      <graphics/>
    </devices>
  </domain>
  <storage>
    <disk file='centos-base-sda.raw' use='system' format='raw'/>
  </storage>
</image>
"""

import xmlrpclib
class XCPError(Exception):
    def __init__(self, result):
        self.result = result
    def __str__(self):
        # {'Status': 'Failure', 'ErrorDescription': ['SESSION_AUTHENTICATION_FAILED', 'a', 'Authentication failure']}
        return " ".join(self.result["ErrorDescription"])

class Failure(Exception):
    def __init__(self, reason):
        self.reason = reason
    def __str__(self):
        return self.reason

def value(x):
    if "Value" in x:
        return x["Value"]
    else:
        raise XCPError(x)

# We base our VMs off this generic HVM template
base_template = "Other install media"

import xml.dom.minidom
import sys

# Creates the VM, VBDs and VDIs
def import_metadata(server, session, filename):
    doc = xml.dom.minidom.parse(filename)
    def getSingleElement(doc, name):
        elements = doc.getElementsByTagName(name)
        if len(elements) <> 1:
            raise Failure("Expecting exactly one <%s> element" % name)
        return elements[0]
    image = getSingleElement(doc, "image")
    domain = getSingleElement(image, "domain")
    boot = getSingleElement(domain, "boot")
    devices = getSingleElement(domain, "devices")
    storage = getSingleElement(image, "storage")
    def getText(doc, name):
        nodes = doc.getElementsByTagName(name)
        if len(nodes) <> 1:
            print >>sys.stderr, "Expecting exactly one %s tag" % name
            sys.exit(1)
        result = ""
        for child in nodes[0].childNodes:
            if child.nodeType == child.TEXT_NODE:
                result = result + child.data
        return result
    def getAttr(doc, name):
        for (n, value) in doc.attributes.items():
            if name == n:
                return value
        return ""
    # Clone the "Other install media" template and inherit basic
    # properties from it.
    templates = value(server.VM.get_by_name_label(session, base_template))
    if len(templates) <> 1:
        raise Failure("Expecting exactly one \"%s\" template" % base_template)
    template = templates[0]
    name = getText(image, "name")
    log("Cloning template %s into %s" % (base_template, name))
    vm = value(server.VM.clone(session, template, name))
    value(server.VM.set_is_a_template(session, vm, False))
    vcpu = getText(devices, "vcpu")
    if vcpu <> "":
        log("Setting number of vCPUs to: %s" % vcpu)
        value(server.VM.set_VCPUs_max(session, vm, vcpu))
        value(server.VM.set_VCPUs_at_startup(session, vm, vcpu))
    memory = getText(devices, "memory") # KiB
    if memory <> "":
        log("Setting memory to %s KiB" % memory)
        bytes = str(long(memory) * 1024L)
        value(server.VM.set_memory_limits(session, vm, "0", bytes, bytes, bytes))
    boot_type = getAttr(boot, "type")
    if boot_type == "hvm":
        log("VM is set to HVM boot by default")
    else:
        log("Ignoring unknown boot type: %s" % boot_type)
    # Disks
    disks = storage.getElementsByTagName("disk")
    drives = boot.getElementsByTagName("drive")
    pool = value(server.pool.get_all(session))[0]
    sr = value(server.pool.get_default_SR(session, pool))
    try:
        log("Will create disks in the default SR: %s" % (value(server.SR.get_name_label(session, sr))))
    except Exception, e:
        log("Caught %s" % str(e))
        raise Failure("Default SR is not set on the pool (%s)" % sr)
    vdis = {}
    for disk in disks:
        ty = getAttr(disk, "format")
        if ty <> "raw":
            raise Failure("Expected all disks to have format = raw")
        filename = getAttr(disk, "file")
        size = os.path.getsize(filename)
        _type = "user"
        if getAttr(disk, "use") == "system":
            _type = "system"
        vdi_info = {
            "name_label": filename,
            "name_description": "",
            "SR": sr,
            "virtual_size": str(size),
            "type": _type,
            "sharable": False,
            "read_only": False,
            "other_config": {},
            }
        vdi = value(server.VDI.create(session, vdi_info))
        log("Created VDI %s for %s" % (vdi, filename))
        vdis[filename] = vdi
    for drive in drives:
        disk = getAttr(drive, "disk")
        target = getAttr(drive, "target")
        vdi = vdis[disk]
        bootable = drive == drives[0]
        vbd_info = {
            "VM": vm,
            "VDI": vdi,
            "userdevice": target,
            "bootable": bootable,
            "mode": "RW",
            "type": "Disk",
            "empty": False,
            "other_config": { "owner": "true" },
            "qos_algorithm_type": "",
            "qos_algorithm_params": {},
            }
        vbd = value(server.VBD.create(session, vbd_info))
        log("Created VBD %s for %s" % (vbd, disk))
    return (vm, vdis)

CURL = "/usr/bin/curl"
if not(os.path.exists(CURL)):
    raise Failure("%s doesn't exist" % CURL)

import commands
def import_vdi(url, session, vdi, filename):
    cmd = "%s -T%s %s/import_raw_vdi?session_id=%s\&vdi=%s" % (CURL, filename, url, session, vdi)
    log("%s" % cmd)
    (code, output) = commands.getstatusoutput(cmd)
    if code <> 0:
        log("Disk upload failed: %s" % output)
        raise Failure("disk upload failed")

if __name__ == "__main__":
    from optparse import OptionParser

    settings = {
        "log": "stdout:",
        "server": "http://127.0.0.1",
        "username": "root",
        "password": "",
        }

    log("settings = %s" % repr(settings))
    
    parser = OptionParser(usage="usage: %prog [options] filename.xml")
    parser.add_option("-l", "--log", dest="logfile", help="log to LOG", metavar="LOG")
    parser.add_option("-s", "--server", dest="server", help="connect to SERVER", metavar="SERVER")
    parser.add_option("-u", "--username", dest="username", help="login as USERNAME", metavar="USERNAME")
    parser.add_option("-p", "--password", dest="password", help="use password PASSWORD", metavar="PASSWORD")
    (options, args) = parser.parse_args()
    if len(args) != 1:
        parser.error("wrong number of arguments")

    options = options.__dict__

    for setting in settings:
        if setting in options and options[setting]:
            settings[setting] = options[setting]
            s = repr(settings[setting])
            if setting == "password":
                s = "*REDACTED*"
            log("option settings[%s] <- %s" % (setting, s))

    if settings["log"] == "syslog:":
        use_syslog = True
        reopenlog(None)
    elif settings["log"] == "stdout:":
        use_syslog = False
        reopenlog("stdout:")
    else:
        use_syslog = False
        reopenlog(settings["log"])

    server = xmlrpclib.Server(settings["server"])
    session = value(server.session.login_with_password(settings["username"], settings["password"]))
    try:
        (vm, vdis) = import_metadata(server, session, args[0])
        for filename in vdis.keys():
            import_vdi(settings["server"], session, vdis[filename], filename)
        log("VM import complete")
        log("%s" % vm)
    finally:
        value(server.session.logout(session))

{'fraction_non_alphanumeric': 0.1374625588361147, 'fraction_numerical': 0.0036371416345742404, 'mean_word_length': 2.6850610957824204, 'pattern_counts': {'":': 34, '<': 40, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '333542', 'n_tokens_mistral': 2997, 'n_tokens_neox': 2919, 'n_words': 811}