<commit_before>from __future__ import print_function
import xml.etree.ElementTree as ET
import glob
import libvirt
import os
from string import (Template, lowercase)
import sys
import subprocess
import itertools as it

TEMPLATE_DIR = "{0}/templates".format(os.path.dirname(__file__))


def libvirt_connect():
    conn = libvirt.open("qemu:///system")
    if not conn:
        print("Failed to open connection to the hypervisor")
        sys.exit(1)
    return conn


def readfile(fname):
    with open(os.path.join(os.path.dirname(__file__), fname)) as f:
        ret = f.read()
    return ret


def remove_files(files):
    for f in glob.glob(files):
        print("removing {0}".format(f))
        os.remove(f)


def cpuflags():
    cpu_flags = readfile("{0}/cpu-default.xml".format(TEMPLATE_DIR))
    if (subprocess.call(["grep", "-q", "flags.* npt", "/proc/cpuinfo"]) == 0):
        cpu_flags = ""
    if (subprocess.call(["grep", "-q", "vendor_id.*GenuineIntel",
                        "/proc/cpuinfo"]) == 0):
        cpu_flags = readfile("{0}/cpu-intel.xml".format(TEMPLATE_DIR))
    return cpu_flags


def hypervisor_has_virtio(libvirt_type):
    if (libvirt_type == "xen" or libvirt_type == "hyperv"):
        return False
    else:
        return True


def get_config(values, fin):
    template = Template(readfile(fin))
    return template.substitute(values)


def admin_config(args, cpu_flags=cpuflags()):
    fin = "{0}/admin-node.xml".format(TEMPLATE_DIR)

    # add xml snippet to be able to mount a local dir via 9p in a VM
    localrepomount = ""
    if args.localreposrc and args.localrepotgt:
        local_repo_template = Template(readfile(
            "{0}/local-repository-mount.xml".format(TEMPLATE_DIR)))
        local_repo_values = dict(localreposdir_src=args.localreposrc,
                                 localreposdir_target=args.localrepotgt)
        localrepomount = local_repo_template.substitute(local_repo_values)

    values = dict(
        cloud=args.cloud,
        admin_node_memory=args.adminnodememory,
        adminvcpus=args.adminvcpus,
        cpuflags=cpu_flags,
        emulator=args.emulator,
        admin_node_disk=args.adminnodedisk,
        local_repository_mount=localrepomount)

    return get_config(values, fin)


def net_config(args):
    cloud = args.cloud
    fin = "{0}/admin-net.xml".format(TEMPLATE_DIR)
    values = dict(
        cloud=cloud,
        cloudbr=args.cloudbr,
        admingw=args.admingw,
        adminnetmask=args.adminnetmask,
        cloudfqdn=args.cloudfqdn,
        adminip=args.adminip,
        forwardmode=args.forwardmode)

    return get_config(values, fin)


def compute_config(args, cpu_flags=cpuflags()):
    fin = "{0}/compute-node.xml".format(TEMPLATE_DIR)
    libvirt_type = args.libvirttype
    alldevices = it.chain(it.chain(lowercase[1:]),
                          it.product(lowercase, lowercase))

    if hypervisor_has_virtio(libvirt_type):
        nicmodel = "virtio"
        targetdevprefix = "vd"
        targetbus = "virtio"
    else:
        nicmodel = "e1000"
        targetdevprefix = "sd"
        targetbus = "ide"
    if args.nodecounter == "1":
        nodememory = args.computenodememory
    else:
        nodememory = args.controllernodememory

    cephvolume = ""
    if args.cephvolumenumber and args.cephvolumenumber > 0:
        for i in range(1, int(args.cephvolumenumber) + 1):
            ceph_template = Template(readfile(
                "{0}/extra-volume.xml".format(TEMPLATE_DIR)))
            ceph_values = dict(
                volume_serial="{0}-node{1}-ceph{2}".format(
                    args.cloud,
                    args.nodecounter,
                    i),
                source_dev="{0}/{1}.node{2}-ceph{3}".format(
                    args.vdiskdir,
                    args.cloud,
                    args.nodecounter,
                    i),
                target_dev=targetdevprefix + ''.join(alldevices.next()),
                target_bus=targetbus)
            cephvolume += "\n" + ceph_template.substitute(ceph_values)

    drbdvolume = ""
    if args.drbdserial:
        drbd_template = Template(readfile(
            "{0}/extra-volume.xml".format(TEMPLATE_DIR)))
        drbd_values = dict(
            volume_serial=args.drbdserial,
            source_dev="{0}/{1}.node{2}-drbd".format(
                args.vdiskdir,
                args.cloud,
                args.nodecounter),
            target_dev=targetdevprefix + ''.join(alldevices.next()),
            target_bus=targetbus)
        drbdvolume = drbd_template.substitute(drbd_values)

    values = dict(
        cloud=args.cloud,
        nodecounter=args.nodecounter,
        nodememory=nodememory,
        vcpus=args.vcpus,
        cpuflags=cpu_flags,
        emulator=args.emulator,
        vdisk_dir=args.vdiskdir,
        cephvolume=cephvolume,
        drbdvolume=drbdvolume,
        macaddress=args.macaddress,
        nicmodel=nicmodel,
        target_dev=targetdevprefix + 'a',
        target_bus=targetbus,
        bootorder=args.bootorder)

    return get_config(values, fin)


def cleanup(args):
    conn = libvirt_connect()
    devnull = open(os.devnull, "w")
    domains = [i for i in conn.listAllDomains()
               if i.name().startswith(args.cloud)]

    for dom in domains:
        domain_cleanup(dom)
        machine = "{0}-{1}".format("qemu", dom.name())
        try:
            machine_status = subprocess.call(
                ["machinectl", "status", machine],
                stdout=devnull, stderr=subprocess.STDOUT)
            if machine_status == 0:
                # workaround bnc#916518
                print("Working around bnc#916518 by \
                      'machinectl terminate {0}'".format(machine))
                subprocess.call(["machinectl", "terminate", machine])
        except OSError:
            # no machinectl available
            pass

    networks = [i for i in conn.listAllNetworks()
                if i.name() == "{0}-admin".format(args.cloud)]
    for network in networks:
        if network.isActive():
            print("destroying {0}".format(network.name()))
            network.destroy()
<commit_msg>fixup! libvirt_setup.py: a python module for libvirt resource handling<commit_after>from __future__ import print_function
import xml.etree.ElementTree as ET
import glob
import libvirt
import os
from string import (Template, lowercase)
import sys
import subprocess
import itertools as it

TEMPLATE_DIR = "{0}/templates".format(os.path.dirname(__file__))


def libvirt_connect():
    conn = libvirt.open("qemu:///system")
    if not conn:
        print("Failed to open connection to the hypervisor")
        sys.exit(1)
    return conn


def readfile(fname):
    with open(os.path.join(os.path.dirname(__file__), fname)) as f:
        ret = f.read()
    return ret


def remove_files(files):
    for f in glob.glob(files):
        print("removing {0}".format(f))
        os.remove(f)


def cpuflags():
    cpu_flags = readfile("{0}/cpu-default.xml".format(TEMPLATE_DIR))
    if (subprocess.call(["grep", "-q", "flags.* npt", "/proc/cpuinfo"]) == 0):
        cpu_flags = ""
    if (subprocess.call(["grep", "-q", "vendor_id.*GenuineIntel",
                        "/proc/cpuinfo"]) == 0):
        cpu_flags = readfile("{0}/cpu-intel.xml".format(TEMPLATE_DIR))
    return cpu_flags


def hypervisor_has_virtio(libvirt_type):
    if (libvirt_type == "xen" or libvirt_type == "hyperv"):
        return False
    else:
        return True


def get_config(values, fin):
    template = Template(readfile(fin))
    return template.substitute(values)


def admin_config(args, cpu_flags=cpuflags()):
    fin = "{0}/admin-node.xml".format(TEMPLATE_DIR)

    # add xml snippet to be able to mount a local dir via 9p in a VM
    localrepomount = ""
    if args.localreposrc and args.localrepotgt:
        local_repo_template = Template(readfile(
            "{0}/local-repository-mount.xml".format(TEMPLATE_DIR)))
        local_repo_values = dict(localreposdir_src=args.localreposrc,
                                 localreposdir_target=args.localrepotgt)
        localrepomount = local_repo_template.substitute(local_repo_values)

    values = dict(
        cloud=args.cloud,
        admin_node_memory=args.adminnodememory,
        adminvcpus=args.adminvcpus,
        cpuflags=cpu_flags,
        emulator=args.emulator,
        admin_node_disk=args.adminnodedisk,
        local_repository_mount=localrepomount)

    return get_config(values, fin)


def net_config(args):
    cloud = args.cloud
    fin = "{0}/admin-net.xml".format(TEMPLATE_DIR)
    values = dict(
        cloud=cloud,
        cloudbr=args.cloudbr,
        admingw=args.admingw,
        adminnetmask=args.adminnetmask,
        cloudfqdn=args.cloudfqdn,
        adminip=args.adminip,
        forwardmode=args.forwardmode)

    return get_config(values, fin)


def compute_config(args, cpu_flags=cpuflags()):
    fin = "{0}/compute-node.xml".format(TEMPLATE_DIR)
    libvirt_type = args.libvirttype
    alldevices = it.chain(it.chain(lowercase[1:]),
                          it.product(lowercase, lowercase))

    if hypervisor_has_virtio(libvirt_type):
        nicmodel = "virtio"
        targetdevprefix = "vd"
        targetbus = "virtio"
    else:
        nicmodel = "e1000"
        targetdevprefix = "sd"
        targetbus = "ide"
    if args.nodecounter == "1":
        nodememory = args.computenodememory
    else:
        nodememory = args.controllernodememory

    cephvolume = ""
    if args.cephvolumenumber and args.cephvolumenumber > 0:
        for i in range(1, int(args.cephvolumenumber) + 1):
            ceph_template = Template(readfile(
                "{0}/extra-volume.xml".format(TEMPLATE_DIR)))
            ceph_values = dict(
                volume_serial="{0}-node{1}-ceph{2}".format(
                    args.cloud,
                    args.nodecounter,
                    i),
                source_dev="{0}/{1}.node{2}-ceph{3}".format(
                    args.vdiskdir,
                    args.cloud,
                    args.nodecounter,
                    i),
                target_dev=targetdevprefix + ''.join(alldevices.next()),
                target_bus=targetbus)
            cephvolume += "\n" + ceph_template.substitute(ceph_values)

    drbdvolume = ""
    if args.drbdserial:
        drbd_template = Template(readfile(
            "{0}/extra-volume.xml".format(TEMPLATE_DIR)))
        drbd_values = dict(
            volume_serial=args.drbdserial,
            source_dev="{0}/{1}.node{2}-drbd".format(
                args.vdiskdir,
                args.cloud,
                args.nodecounter),
            target_dev=targetdevprefix + ''.join(alldevices.next()),
            target_bus=targetbus)
        drbdvolume = drbd_template.substitute(drbd_values)

    values = dict(
        cloud=args.cloud,
        nodecounter=args.nodecounter,
        nodememory=nodememory,
        vcpus=args.vcpus,
        cpuflags=cpu_flags,
        emulator=args.emulator,
        vdisk_dir=args.vdiskdir,
        cephvolume=cephvolume,
        drbdvolume=drbdvolume,
        macaddress=args.macaddress,
        nicmodel=nicmodel,
        target_dev=targetdevprefix + 'a',
        target_bus=targetbus,
        bootorder=args.bootorder)

    return get_config(values, fin)


def cleanup(args):
    conn = libvirt_connect()
    devnull = open(os.devnull, "w")
    domains = [i for i in conn.listAllDomains()
               if i.name().startswith(args.cloud+"-")]

    for dom in domains:
        domain_cleanup(dom)
        machine = "{0}-{1}".format("qemu", dom.name())
        try:
            machine_status = subprocess.call(
                ["machinectl", "status", machine],
                stdout=devnull, stderr=subprocess.STDOUT)
            if machine_status == 0:
                # workaround bnc#916518
                print("Working around bnc#916518 by \
                      'machinectl terminate {0}'".format(machine))
                subprocess.call(["machinectl", "terminate", machine])
        except OSError:
            # no machinectl available
            pass

    networks = [i for i in conn.listAllNetworks()
                if i.name() == "{0}-admin".format(args.cloud)]
    for network in networks:
        if network.isActive():
            print("destroying {0}".format(network.name()))
            network.destroy()

{'fraction_non_alphanumeric': 0.11587775179421014, 'fraction_numerical': 0.008225143133618256, 'mean_word_length': 2.288782816229117, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5612123', 'n_tokens_mistral': 3976, 'n_tokens_neox': 3829, 'n_words': 700}