<commit_before># Copyright (C) 2010,2011  <NAME> <<EMAIL>>

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

import uuid
import libvirt
import os
import subprocess
import shutil
import time
import pycurl
import urllib2
import stat
import ozutil
import libxml2
import logging
import random
import guestfs
import socket
import select
import struct
import numpy

class ProcessError(Exception):
    """This exception is raised when a process run by
    Guest.subprocess_check_output returns a non-zero exit status.  The exit
    status will be stored in the returncode attribute
    """
    def __init__(self, returncode, cmd, output=None):
        self.returncode = returncode
        self.cmd = cmd
        self.output = output
    def __str__(self):
        return "'%s' failed(%d): %s" % (self.cmd, self.returncode, self.output)

def libvirt_error_handler(ctxt, err):
    pass

# NOTE: python 2.7 already defines subprocess.capture_output, but I can't
# depend on that yet.  So write my own
def subprocess_check_output(*popenargs, **kwargs):
    if 'stdout' in kwargs:
        raise ValueError('stdout argument not allowed, it will be overridden.')

    ozutil.executable_exists(popenargs[0][0])

    # FIXME: this doesn't seem to do the right thing with respect to stderr
    # and reporting errors.  Needs to be looked into
    process = subprocess.Popen(stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT, *popenargs, **kwargs)
    output, unused_err = process.communicate()
    retcode = process.poll()
    if retcode:
        cmd = ' '.join(*popenargs)
        raise ProcessError(retcode, cmd, output=output)
    return output

class OzException(Exception):
    def __init__(self, msg):
        self.msg = msg
    def __str__(self):
        return self.msg

class Guest(object):
    def __init__(self, distro, update, arch, nicmodel, clockoffset, mousetype,
                 diskbus, config):
        if arch != "i386" and arch != "x86_64":
            raise OzException("Unsupported guest arch " + arch)
        self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
        self.uuid = uuid.uuid4()
        mac = [0x52, 0x54, 0x00, random.randint(0x00, 0xff),
               random.randint(0x00, 0xff), random.randint(0x00, 0xff)]
        self.macaddr = ':'.join(map(lambda x:"%02x" % x, mac))
        self.distro = distro
        self.update = update
        self.arch = arch
        self.name = self.distro + self.update + self.arch

        if config is not None and config.has_section('paths') and config.has_option('paths', 'output_dir'):
            self.output_dir = config.get('paths', 'output_dir')
        else:
            self.output_dir = "/var/lib/libvirt/images"

        if config is not None and config.has_section('paths') and config.has_option('paths', 'data_dir'):
            self.data_dir = config.get('paths', 'data_dir')
        else:
            self.data_dir = "/var/lib/oz"

        self.diskimage = self.output_dir + "/" + self.name + ".dsk"
        self.icicle_tmp = self.data_dir + "/icicletmp/" + self.name
        self.listen_port = random.randrange(1024, 65535)
        libvirt.registerErrorHandler(libvirt_error_handler, 'context')
        self.libvirt_conn = libvirt.open("qemu:///system")

        # we have to make sure that the private libvirt bridge is available
        self.host_bridge_ip = None
        for netname in self.libvirt_conn.listNetworks():
            network = self.libvirt_conn.networkLookupByName(netname)
            if network.bridgeName() == 'virbr0':
                xml = network.XMLDesc(0)
                doc = libxml2.parseMemory(xml, len(xml))
                ip = doc.xpathEval('/network/ip')
                if len(ip) != 1:
                    raise OzException("Failed to find host IP address for virbr0")
                self.host_bridge_ip = ip[0].prop('address')
                break
        if self.host_bridge_ip is None:
            raise OzException("Default libvirt network (virbr0) does not exist, install cannot continue")

        self.nicmodel = nicmodel
        if self.nicmodel is None:
            self.nicmodel = "rtl8139"
        self.clockoffset = clockoffset
        if self.clockoffset is None:
            self.clockoffset = "utc"
        self.mousetype = mousetype
        if self.mousetype is None:
            self.mousetype = "ps2"
        if diskbus is None or diskbus == "ide":
            self.disk_bus = "ide"
            self.disk_dev = "hda"
        elif diskbus == "virtio":
            self.disk_bus = "virtio"
            self.disk_dev = "vda"
        else:
            raise OzException("Unknown diskbus type " + diskbus)

        self.log.debug("Name: %s, UUID: %s" % (self.name, self.uuid))
        self.log.debug("MAC: %s, distro: %s" % (self.macaddr, self.distro))
        self.log.debug("update: %s, arch: %s, diskimage: %s" % (self.update, self.arch, self.diskimage))
        self.log.debug("host IP: %s, nicmodel: %s, clockoffset: %s" % (self.host_bridge_ip, self.nicmodel, self.clockoffset))
        self.log.debug("mousetype: %s, disk_bus: %s, disk_dev: %s" % (self.mousetype, self.disk_bus, self.disk_dev))
        self.log.debug("icicletmp: %s, listen_port: %d" % (self.icicle_tmp, self.listen_port))

    def cleanup_old_guest(self, delete_disk=True):
        self.log.info("Cleaning up guest named %s" % (self.name))
        try:
            dom = self.libvirt_conn.lookupByName(self.name)
            try:
                dom.destroy()
            except:
                pass
            dom.undefine()
        except:
            pass

        if delete_disk and os.access(self.diskimage, os.F_OK):
            os.unlink(self.diskimage)

    # the next 3 methods are intended to be overridden by the individual
    # OS backends; raise an error if they are called but not implemented
    def generate_install_media(self, force_download):
        raise OzException("Install media for %s is not implemented, install cannot continue" % (self.name))

    def customize(self, libvirt_xml):
        raise OzException("Customization for %s is not implemented" % (self.name))

    def generate_icicle(self, libvirt_xml):
        raise OzException("ICICLE generation for %s is not implemented" % (self.name))

    def targetDev(self, doc, devicetype, path, bus):
        install = doc.newChild(None, "disk", None)
        install.setProp("type", "file")
        install.setProp("device", devicetype)
        source = install.newChild(None, "source", None)
        source.setProp("file", path)
        target = install.newChild(None, "target", None)
        target.setProp("dev", bus)

    def generate_xml(self, bootdev, want_install_disk=True):
        self.log.info("Generate XML for guest %s with bootdev %s" % (self.name, bootdev))

        # create XML document
        doc = libxml2.newDoc("1.0")

        # create top-level domain element
        domain = doc.newChild(None, "domain", None)
        domain.setProp("type", "kvm")

        # create name element
        name = domain.newChild(None, "name", self.name)

        # create memory elements
        memory = domain.newChild(None, "memory", "1048576")
        currentMemory = domain.newChild(None, "currentMemory", "1048576")

        # create uuid
        uuid = domain.newChild(None, "uuid", str(self.uuid))

        # create clock offset
        clock = domain.newChild(None, "clock", None)
        clock.setProp("offset", self.clockoffset)

        # create vcpu
        vcpu = domain.newChild(None, "vcpu", "1")

        # create features
        features = domain.newChild(None, "features", None)
        acpi = features.newChild(None, "acpi", None)
        apic = features.newChild(None, "apic", None)
        pae = features.newChild(None, "pae", None)

        # create os
        osNode = domain.newChild(None, "os", None)
        ostype = osNode.newChild(None, "type", "hvm")
        boot = osNode.newChild(None, "boot", None)
        boot.setProp("dev", bootdev)

        # create poweroff, reboot, crash
        poweroff = domain.newChild(None, "on_poweroff", "destroy")
        reboot = domain.newChild(None, "on_reboot", "destroy")
        crash = domain.newChild(None, "on_crash", "destroy")

        # create devices
        devices = domain.newChild(None, "devices", None)
        # console
        console = devices.newChild(None, "console", None)
        console.setProp("device", "pty")
        # graphics
        graphics = devices.newChild(None, "graphics", None)
        graphics.setProp("port", "-1")
        graphics.setProp("type", "vnc")
        # network
        interface = devices.newChild(None, "interface", None)
        interface.setProp("type", "bridge")
        interfaceSource = interface.newChild(None, "source", None)
        interfaceSource.setProp("bridge", "virbr0")
        interfaceMac = interface.newChild(None, "mac", None)
        interfaceMac.setProp("address", self.macaddr)
        interfaceModel = interface.newChild(None, "model", None)
        interfaceModel.setProp("type", self.nicmodel)
        # input
        inputdev = devices.newChild(None, "input", None)
        if self.mousetype == "ps2":
            inputdev.setProp("bus", "ps2")
            inputdev.setProp("type", "mouse")
        elif self.mousetype == "usb":
            inputdev.setProp("type", "tablet")
            inputdev.setProp("bus", "usb")
        # console
        console = devices.newChild(None, "console", None)
        console.setProp("type", "pty")
        consoleTarget = console.newChild(None, "target", None)
        consoleTarget.setProp("port", "0")
        # boot disk
        bootDisk = devices.newChild(None, "disk", None)
        bootDisk.setProp("device", "disk")
        bootDisk.setProp("type", "file")
        bootTarget = bootDisk.newChild(None, "target", None)
        bootTarget.setProp("dev", self.disk_dev)
        bootTarget.setProp("bus", self.disk_bus)
        bootSource = bootDisk.newChild(None, "source", None)
        bootSource.setProp("file", self.diskimage)
        # install disk (cdrom or floppy)
        if want_install_disk:
            if hasattr(self, "output_iso"):
                self.targetDev(devices, "cdrom", self.output_iso, "hdc")
            if hasattr(self, "output_floppy"):
                self.targetDev(devices, "floppy", self.output_floppy, "fda")

        self.log.debug("Generated XML:\n%s" % (doc.serialize(None, 1)))

        return doc.serialize(None, 1)

    def generate_blank_diskimage(self, size=10):
        self.log.info("Generating %dGB blank diskimage for %s" % (size, self.name))
        f = open(self.diskimage, "w")
        # 10 GB disk image by default
        f.truncate(size * 1024 * 1024 * 1024)
        f.close()

    def generate_diskimage(self, size=10):
        self.log.info("Generating %dGB diskimage with fake partition for %s" % (size, self.name))
        # FIXME: I think that this partition table will only work with the 10GB
        # image.  We'll need to do something more sophisticated when we handle
        # variable sized disks
        f = open(self.diskimage, "w")
        f.seek(0x1bf)
        f.write("\x01\x01\x00\x82\xfe\x3f\x7c\x3f\x00\x00\x00\xfe\xa3\x1e")
        f.seek(0x1fe)
        f.write("\x55\xaa")
        f.seek(size * 1024 * 1024 * 1024)
        f.write("\x00")
        f.close()

    def wait_for_install_finish(self, libvirt_dom, count):
        origcount = count
        while count > 0:
            if count % 10 == 0:
                self.log.debug("Waiting for %s to finish installing, %d/%d" % (self.name, count, origcount))
            try:
                info = libvirt_dom.info()
            except libvirt.libvirtError, e:
                self.log.debug("Libvirt Domain Info Failed:")
                self.log.debug(" code is %d" % e.get_error_code())
                self.log.debug(" domain is %d" % e.get_error_domain())
                self.log.debug(" message is %s" % e.get_error_message())
                self.log.debug(" level is %d" % e.get_error_level())
                self.log.debug(" str1 is %s" % e.get_str1())
                self.log.debug(" str2 is %s" % e.get_str2())
                self.log.debug(" str3 is %s" % e.get_str3())
                self.log.debug(" int1 is %d" % e.get_int1())
                self.log.debug(" int2 is %d" % e.get_int2())
                if e.get_error_domain() == libvirt.VIR_FROM_QEMU and (e.get_error_code() in [libvirt.VIR_ERR_NO_DOMAIN, libvirt.VIR_ERR_SYSTEM_ERROR]):
                    break
                else:
                    raise

            count -= 1
            time.sleep(1)

        if count == 0:
            # if we timed out, then let's make sure to take a screenshot.
            # FIXME: where should we put this screenshot?
            screenshot = self.name + "-" + str(time.time()) + ".png"
            self.capture_screenshot(libvirt_dom.XMLDesc(0), screenshot)
            raise OzException("Timed out waiting for install to finish")

        self.log.info("Install of %s succeeded" % (self.name))

    def wait_for_guest_shutdown(self, libvirt_dom, count=60):
        origcount = count
        while count > 0:
            if count % 10 == 0:
                self.log.debug("Waiting for %s to shutdown, %d/%d" % (self.name, count, origcount))
            try:
                info = libvirt_dom.info()
            except libvirt.libvirtError, e:
                self.log.debug("Libvirt Domain Info Failed:")
                self.log.debug(" code is %d" % e.get_error_code())
                self.log.debug(" domain is %d" % e.get_error_domain())
                self.log.debug(" message is %s" % e.get_error_message())
                self.log.debug(" level is %d" % e.get_error_level())
                self.log.debug(" str1 is %s" % e.get_str1())
                self.log.debug(" str2 is %s" % e.get_str2())
                self.log.debug(" str3 is %s" % e.get_str3())
                self.log.debug(" int1 is %d" % e.get_int1())
                self.log.debug(" int2 is %d" % e.get_int2())
                if e.get_error_domain() == libvirt.VIR_FROM_QEMU and (e.get_error_code() in [libvirt.VIR_ERR_NO_DOMAIN, libvirt.VIR_ERR_SYSTEM_ERROR]):
                    break
                else:
                    raise

            count -= 1
            time.sleep(1)

        return count != 0

    def get_original_media(self, url, output, force_download):
        self.log.info("Fetching the original media")

        try:
            response = urllib2.urlopen(url)
            url = response.geturl()
            content_length = int(response.info()["Content-Length"])
            response.close()<commit_msg>Catch VIR_ERR_OPERATION_FAILED error as well.<commit_after># Copyright (C) 2010,2011  <NAME> <<EMAIL>>

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

import uuid
import libvirt
import os
import subprocess
import shutil
import time
import pycurl
import urllib2
import stat
import ozutil
import libxml2
import logging
import random
import guestfs
import socket
import select
import struct
import numpy

class ProcessError(Exception):
    """This exception is raised when a process run by
    Guest.subprocess_check_output returns a non-zero exit status.  The exit
    status will be stored in the returncode attribute
    """
    def __init__(self, returncode, cmd, output=None):
        self.returncode = returncode
        self.cmd = cmd
        self.output = output
    def __str__(self):
        return "'%s' failed(%d): %s" % (self.cmd, self.returncode, self.output)

def libvirt_error_handler(ctxt, err):
    pass

# NOTE: python 2.7 already defines subprocess.capture_output, but I can't
# depend on that yet.  So write my own
def subprocess_check_output(*popenargs, **kwargs):
    if 'stdout' in kwargs:
        raise ValueError('stdout argument not allowed, it will be overridden.')

    ozutil.executable_exists(popenargs[0][0])

    # FIXME: this doesn't seem to do the right thing with respect to stderr
    # and reporting errors.  Needs to be looked into
    process = subprocess.Popen(stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT, *popenargs, **kwargs)
    output, unused_err = process.communicate()
    retcode = process.poll()
    if retcode:
        cmd = ' '.join(*popenargs)
        raise ProcessError(retcode, cmd, output=output)
    return output

class OzException(Exception):
    def __init__(self, msg):
        self.msg = msg
    def __str__(self):
        return self.msg

class Guest(object):
    def __init__(self, distro, update, arch, nicmodel, clockoffset, mousetype,
                 diskbus, config):
        if arch != "i386" and arch != "x86_64":
            raise OzException("Unsupported guest arch " + arch)
        self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
        self.uuid = uuid.uuid4()
        mac = [0x52, 0x54, 0x00, random.randint(0x00, 0xff),
               random.randint(0x00, 0xff), random.randint(0x00, 0xff)]
        self.macaddr = ':'.join(map(lambda x:"%02x" % x, mac))
        self.distro = distro
        self.update = update
        self.arch = arch
        self.name = self.distro + self.update + self.arch

        if config is not None and config.has_section('paths') and config.has_option('paths', 'output_dir'):
            self.output_dir = config.get('paths', 'output_dir')
        else:
            self.output_dir = "/var/lib/libvirt/images"

        if config is not None and config.has_section('paths') and config.has_option('paths', 'data_dir'):
            self.data_dir = config.get('paths', 'data_dir')
        else:
            self.data_dir = "/var/lib/oz"

        self.diskimage = self.output_dir + "/" + self.name + ".dsk"
        self.icicle_tmp = self.data_dir + "/icicletmp/" + self.name
        self.listen_port = random.randrange(1024, 65535)
        libvirt.registerErrorHandler(libvirt_error_handler, 'context')
        self.libvirt_conn = libvirt.open("qemu:///system")

        # we have to make sure that the private libvirt bridge is available
        self.host_bridge_ip = None
        for netname in self.libvirt_conn.listNetworks():
            network = self.libvirt_conn.networkLookupByName(netname)
            if network.bridgeName() == 'virbr0':
                xml = network.XMLDesc(0)
                doc = libxml2.parseMemory(xml, len(xml))
                ip = doc.xpathEval('/network/ip')
                if len(ip) != 1:
                    raise OzException("Failed to find host IP address for virbr0")
                self.host_bridge_ip = ip[0].prop('address')
                break
        if self.host_bridge_ip is None:
            raise OzException("Default libvirt network (virbr0) does not exist, install cannot continue")

        self.nicmodel = nicmodel
        if self.nicmodel is None:
            self.nicmodel = "rtl8139"
        self.clockoffset = clockoffset
        if self.clockoffset is None:
            self.clockoffset = "utc"
        self.mousetype = mousetype
        if self.mousetype is None:
            self.mousetype = "ps2"
        if diskbus is None or diskbus == "ide":
            self.disk_bus = "ide"
            self.disk_dev = "hda"
        elif diskbus == "virtio":
            self.disk_bus = "virtio"
            self.disk_dev = "vda"
        else:
            raise OzException("Unknown diskbus type " + diskbus)

        self.log.debug("Name: %s, UUID: %s" % (self.name, self.uuid))
        self.log.debug("MAC: %s, distro: %s" % (self.macaddr, self.distro))
        self.log.debug("update: %s, arch: %s, diskimage: %s" % (self.update, self.arch, self.diskimage))
        self.log.debug("host IP: %s, nicmodel: %s, clockoffset: %s" % (self.host_bridge_ip, self.nicmodel, self.clockoffset))
        self.log.debug("mousetype: %s, disk_bus: %s, disk_dev: %s" % (self.mousetype, self.disk_bus, self.disk_dev))
        self.log.debug("icicletmp: %s, listen_port: %d" % (self.icicle_tmp, self.listen_port))

    def cleanup_old_guest(self, delete_disk=True):
        self.log.info("Cleaning up guest named %s" % (self.name))
        try:
            dom = self.libvirt_conn.lookupByName(self.name)
            try:
                dom.destroy()
            except:
                pass
            dom.undefine()
        except:
            pass

        if delete_disk and os.access(self.diskimage, os.F_OK):
            os.unlink(self.diskimage)

    # the next 3 methods are intended to be overridden by the individual
    # OS backends; raise an error if they are called but not implemented
    def generate_install_media(self, force_download):
        raise OzException("Install media for %s is not implemented, install cannot continue" % (self.name))

    def customize(self, libvirt_xml):
        raise OzException("Customization for %s is not implemented" % (self.name))

    def generate_icicle(self, libvirt_xml):
        raise OzException("ICICLE generation for %s is not implemented" % (self.name))

    def targetDev(self, doc, devicetype, path, bus):
        install = doc.newChild(None, "disk", None)
        install.setProp("type", "file")
        install.setProp("device", devicetype)
        source = install.newChild(None, "source", None)
        source.setProp("file", path)
        target = install.newChild(None, "target", None)
        target.setProp("dev", bus)

    def generate_xml(self, bootdev, want_install_disk=True):
        self.log.info("Generate XML for guest %s with bootdev %s" % (self.name, bootdev))

        # create XML document
        doc = libxml2.newDoc("1.0")

        # create top-level domain element
        domain = doc.newChild(None, "domain", None)
        domain.setProp("type", "kvm")

        # create name element
        name = domain.newChild(None, "name", self.name)

        # create memory elements
        memory = domain.newChild(None, "memory", "1048576")
        currentMemory = domain.newChild(None, "currentMemory", "1048576")

        # create uuid
        uuid = domain.newChild(None, "uuid", str(self.uuid))

        # create clock offset
        clock = domain.newChild(None, "clock", None)
        clock.setProp("offset", self.clockoffset)

        # create vcpu
        vcpu = domain.newChild(None, "vcpu", "1")

        # create features
        features = domain.newChild(None, "features", None)
        acpi = features.newChild(None, "acpi", None)
        apic = features.newChild(None, "apic", None)
        pae = features.newChild(None, "pae", None)

        # create os
        osNode = domain.newChild(None, "os", None)
        ostype = osNode.newChild(None, "type", "hvm")
        boot = osNode.newChild(None, "boot", None)
        boot.setProp("dev", bootdev)

        # create poweroff, reboot, crash
        poweroff = domain.newChild(None, "on_poweroff", "destroy")
        reboot = domain.newChild(None, "on_reboot", "destroy")
        crash = domain.newChild(None, "on_crash", "destroy")

        # create devices
        devices = domain.newChild(None, "devices", None)
        # console
        console = devices.newChild(None, "console", None)
        console.setProp("device", "pty")
        # graphics
        graphics = devices.newChild(None, "graphics", None)
        graphics.setProp("port", "-1")
        graphics.setProp("type", "vnc")
        # network
        interface = devices.newChild(None, "interface", None)
        interface.setProp("type", "bridge")
        interfaceSource = interface.newChild(None, "source", None)
        interfaceSource.setProp("bridge", "virbr0")
        interfaceMac = interface.newChild(None, "mac", None)
        interfaceMac.setProp("address", self.macaddr)
        interfaceModel = interface.newChild(None, "model", None)
        interfaceModel.setProp("type", self.nicmodel)
        # input
        inputdev = devices.newChild(None, "input", None)
        if self.mousetype == "ps2":
            inputdev.setProp("bus", "ps2")
            inputdev.setProp("type", "mouse")
        elif self.mousetype == "usb":
            inputdev.setProp("type", "tablet")
            inputdev.setProp("bus", "usb")
        # console
        console = devices.newChild(None, "console", None)
        console.setProp("type", "pty")
        consoleTarget = console.newChild(None, "target", None)
        consoleTarget.setProp("port", "0")
        # boot disk
        bootDisk = devices.newChild(None, "disk", None)
        bootDisk.setProp("device", "disk")
        bootDisk.setProp("type", "file")
        bootTarget = bootDisk.newChild(None, "target", None)
        bootTarget.setProp("dev", self.disk_dev)
        bootTarget.setProp("bus", self.disk_bus)
        bootSource = bootDisk.newChild(None, "source", None)
        bootSource.setProp("file", self.diskimage)
        # install disk (cdrom or floppy)
        if want_install_disk:
            if hasattr(self, "output_iso"):
                self.targetDev(devices, "cdrom", self.output_iso, "hdc")
            if hasattr(self, "output_floppy"):
                self.targetDev(devices, "floppy", self.output_floppy, "fda")

        self.log.debug("Generated XML:\n%s" % (doc.serialize(None, 1)))

        return doc.serialize(None, 1)

    def generate_blank_diskimage(self, size=10):
        self.log.info("Generating %dGB blank diskimage for %s" % (size, self.name))
        f = open(self.diskimage, "w")
        # 10 GB disk image by default
        f.truncate(size * 1024 * 1024 * 1024)
        f.close()

    def generate_diskimage(self, size=10):
        self.log.info("Generating %dGB diskimage with fake partition for %s" % (size, self.name))
        # FIXME: I think that this partition table will only work with the 10GB
        # image.  We'll need to do something more sophisticated when we handle
        # variable sized disks
        f = open(self.diskimage, "w")
        f.seek(0x1bf)
        f.write("\x01\x01\x00\x82\xfe\x3f\x7c\x3f\x00\x00\x00\xfe\xa3\x1e")
        f.seek(0x1fe)
        f.write("\x55\xaa")
        f.seek(size * 1024 * 1024 * 1024)
        f.write("\x00")
        f.close()

    def wait_for_install_finish(self, libvirt_dom, count):
        origcount = count
        while count > 0:
            if count % 10 == 0:
                self.log.debug("Waiting for %s to finish installing, %d/%d" % (self.name, count, origcount))
            try:
                info = libvirt_dom.info()
            except libvirt.libvirtError, e:
                self.log.debug("Libvirt Domain Info Failed:")
                self.log.debug(" code is %d" % e.get_error_code())
                self.log.debug(" domain is %d" % e.get_error_domain())
                self.log.debug(" message is %s" % e.get_error_message())
                self.log.debug(" level is %d" % e.get_error_level())
                self.log.debug(" str1 is %s" % e.get_str1())
                self.log.debug(" str2 is %s" % e.get_str2())
                self.log.debug(" str3 is %s" % e.get_str3())
                self.log.debug(" int1 is %d" % e.get_int1())
                self.log.debug(" int2 is %d" % e.get_int2())
                if e.get_error_domain() == libvirt.VIR_FROM_QEMU and (e.get_error_code() in [libvirt.VIR_ERR_NO_DOMAIN, libvirt.VIR_ERR_SYSTEM_ERROR, libvirt.VIR_ERR_OPERATION_FAILED]):
                    break
                else:
                    raise

            count -= 1
            time.sleep(1)

        if count == 0:
            # if we timed out, then let's make sure to take a screenshot.
            # FIXME: where should we put this screenshot?
            screenshot = self.name + "-" + str(time.time()) + ".png"
            self.capture_screenshot(libvirt_dom.XMLDesc(0), screenshot)
            raise OzException("Timed out waiting for install to finish")

        self.log.info("Install of %s succeeded" % (self.name))

    def wait_for_guest_shutdown(self, libvirt_dom, count=60):
        origcount = count
        while count > 0:
            if count % 10 == 0:
                self.log.debug("Waiting for %s to shutdown, %d/%d" % (self.name, count, origcount))
            try:
                info = libvirt_dom.info()
            except libvirt.libvirtError, e:
                self.log.debug("Libvirt Domain Info Failed:")
                self.log.debug(" code is %d" % e.get_error_code())
                self.log.debug(" domain is %d" % e.get_error_domain())
                self.log.debug(" message is %s" % e.get_error_message())
                self.log.debug(" level is %d" % e.get_error_level())
                self.log.debug(" str1 is %s" % e.get_str1())
                self.log.debug(" str2 is %s" % e.get_str2())
                self.log.debug(" str3 is %s" % e.get_str3())
                self.log.debug(" int1 is %d" % e.get_int1())
                self.log.debug(" int2 is %d" % e.get_int2())
                if e.get_error_domain() == libvirt.VIR_FROM_QEMU and (e.get_error_code() in [libvirt.VIR_ERR_NO_DOMAIN, libvirt.VIR_ERR_SYSTEM_ERROR, libvirt.VIR_ERR_OPERATION_FAILED]):
                    break
                else:
                    raise

            count -= 1
            time.sleep(1)

        return count != 0

    def get_original_media(self, url, output, force_download):
        self.log.info("Fetching the original media")

        try:
            response = urllib2.urlopen(url)
            url = response.geturl()
            content_length = int(response.info()["Content-Length"])
            response.close()
{'fraction_non_alphanumeric': 0.11941218451956141, 'fraction_numerical': 0.013105819762538117, 'mean_word_length': 2.347849695916594, 'pattern_counts': {'":': 10, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3979649', 'n_tokens_mistral': 9676, 'n_tokens_neox': 9248, 'n_words': 2477}