<commit_before>#!/usr/bin/python
import libvirt
import argparse
import random
import yaml
import os
import re
import sys
import logging
import subprocess

from xml.etree import ElementTree as et
from tempfile import mkstemp
from libvirt import libvirtError

__author__ = 'sshnaidm'


def rand_mac():
    mac = [0x52, 0x54, 0x00,
           random.randint(0x00, 0xff),
           random.randint(0x00, 0xff),
           random.randint(0x00, 0xff)]
    return ':'.join(["%02x" % x for x in mac])

## TODO: remove GLOBALS
name_woext = "trusty-server-cloudimg-amd64-disk1"
EXTERNAL_NET = "external"
DOMAIN_NAME = "domain.name"


def handler(ctxt, err):
    global errno
    errno = err
libvirt.registerErrorHandler(handler, 'context')


def run_cmd(cmd):
    """
        Return the exit status and output to stdout and stderr.
    """
    logging.debug("Running command: %s" % " ".join(cmd))
    proc = subprocess.Popen(cmd, stderr=subprocess.PIPE,
                            stdout=subprocess.PIPE,
                            close_fds=True)
    ret = proc.wait()
    print >> sys.stderr, "\n".join(proc.stdout.readlines()), "\n".join(proc.stderr.readlines())
    return ret, proc.stdout.readlines(), proc.stderr.readlines()


def vm_create(conn, xml):
    return conn.defineXML(xml)


def net_create(conn, xml):
    net = conn.networkDefineXML(xml)
    net.create()
    net.setAutostart(True)
    return net


def net_undefine(conn, net_name):
    try:
        net = conn.networkLookupByName(net_name)
    except libvirtError:
        print >> sys.stderr, "Net wans't found, nothing to delete"
        return
    try:
        net.destroy()
    except libvirtError:
        print >> sys.stderr, "Net wans't active, nothing to delete"
        return
    net.undefine()


def pool_delete(conn, name):
    try:
        pool = conn.storagePoolLookupByName(name)
    except libvirtError:
        print >> sys.stderr, "Pool not found, nothing to delete"
        return
    try:
        pool.destroy()
    except libvirtError:
        print >> sys.stderr, "Pool wans't active, just undefine it"
    pool.undefine()


def pool_create(conn, xml):
    pool = conn.storagePoolDefineXML(xml)
    pool.create()
    pool.setAutostart(True)
    return pool


def pool_found(conn, name):
    try:
        conn.storagePoolLookupByName(name)
        return True
    except libvirtError:
        return False


def convert_image(path, new_path):
    ret, ignore, stderr = run_cmd(["qemu-img", "convert", "-O",
                                   "qcow2", path, new_path])
    print >> sys.stderr, "\n".join(ignore), "\n".join(stderr)
    if ret == 127:
        ret, ignore, stderr = run_cmd(["kvm-img", "convert", "-O",
                                       "qcow2", path, new_path])
    if ret != 0:
        raise RuntimeError("Disk conversion failed with "
                           "exit status %d: %s" % (ret, "".join(stderr)))
    #if len(stderr):
    #    print >> sys.stderr, stderr


def main_disk_create(name="", size=0, config=None, conn=None, img_path="", boot_type="net", user_seed_yaml=None,
                     img_uncomp_path=None, lab_id="lab1"):
    path = os.path.join(img_path, name + ".qcow2")
    size_bytes = size*1024*1024*1024
    if boot_type == "net":
        vol_xml = config["params"]["vol"]["xml"].format(name=name, size=size_bytes, path=path)
        #_, tmp_file = mkstemp(prefix="disk_vol_")
        #with open(tmp_file, "w") as f:
        #    f.write(vol_xml)
        #run_cmd(["virsh", "vol-create",  "--pool", "default-" + lab_id, "--file", tmp_file])
        pool = conn.storagePoolLookupByName("default-" + lab_id)
        vol = pool.createXML(vol_xml, 1)
        #os.remove(tmp_file)
        return config["params"]["vol"]["virt_disk"].format(output_file=path)
    elif boot_type == "cloudimg":
        seed_disk = os.path.join(img_path, name + "-seed.qcow2")
        #img_uncomp_path=os.path.join(IMG_PATH, "lab-backing.qcow2")
        _, tmp_file = mkstemp(prefix="useryaml_")
        with open(tmp_file, "w") as f:
            f.write("#cloud-config\n" + user_seed_yaml)
        run_cmd(["qemu-img", "create", "-f", "qcow2", "-b", img_uncomp_path, path, "%sG" % size])
        run_cmd(["cloud-localds", seed_disk, tmp_file])
        os.remove(tmp_file)
        return config["params"]["vol"]["cloudimg_disk"].format(output_file=path, seed_disk=seed_disk)


def create_vm(conn, xml):
    vm = conn.defineXML(xml)
    vm.create()
    return vm


def delete_vm(conn, name):
    try:
        vm = conn.lookupByName(name)
    except libvirtError:
        print >> sys.stderr, "Domain {name} not found, nothing to delete".format(name=name)
        return
    try:
        vm.destroy()
    except libvirtError:
        pass
    vm.undefine()
    print >> sys.stderr, "Domain {name} deleted".format(name=name)


def remove_all_imgs(img_path, lab_id):
    if os.path.exists(img_path):
        for i in [j for j in os.listdir(img_path) if lab_id in j]:
            os.remove(os.path.join(img_path, i))


def create_seed_yaml(config, box=None):
    if not box:
        return None
    user_seed_yaml_string = config['params'][box]['user-yaml']
    user_seed_yaml = yaml.load(user_seed_yaml_string)
    user_seed_yaml['users'][1]['ssh-authorized-keys'] = config['params']['id_rsa_pub']
    user_seed_yaml['users'][1]['passwd'] = ("$6$rounds=4096$A0eKrix5oH$4F70Syi4jfhMCRygdOUC.d.qItQ57KsmW8"
                                            "<PASSWORD>r/bPm7ySXdYLoHCdpg3SLlWZlv9FnRAUhgp8C23DiVZr9.")
    user_seed_yaml['write_files'][0]['content'] = "\n".join(config['params']['id_rsa_pub'])
    return yaml.dump(user_seed_yaml)


def create_host_seed_yaml(config, box, btype):
    user_seed_yaml_string = config['params'][btype]['user-yaml']
    user_seed_yaml = yaml.load(user_seed_yaml_string)
    user_seed_yaml['users'][1]['ssh-authorized-keys'] = config['params']['id_rsa_pub']
    user_seed_yaml['users'][1]['passwd'] = ("$6$rounds=4096$A0eKrix5oH$4F70Syi4jfhMCRygdOUC.d.qItQ57KsmW8"
                                            "<PASSWORD>/<PASSWORD>LoHCdpg3SLlWZlv9FnRAUhgp8C23DiVZr9.")
    user_seed_yaml['write_files'][0]['content'] = "\n".join(config['params']['id_rsa_pub'])
    ints = []
    last_octet = box['ip'].split(".")[-1]
    #sorted_nets = sorted(networks.keys(), key=lambda x: int(networks[x]["net-ip"].split(".")[2]))
    sorted_nets = config['params']['networks']
    parsed = et.fromstring(config['params'][btype]['xml'])
    used_nets = [i.attrib['network'] for i in parsed.findall(".//interface[@type='network']/source")]
    index_net = 1
    for k, net in enumerate(sorted_nets[1:]):
        if "{net_" + net['role'] + "_name}" in used_nets:
            if net['role'] != EXTERNAL_NET:
                ints.append([
                    config['params']['static_interface_template'].format(
                        int_name="eth"+str(index_net),
                        net_ip=net['net-ip'],
                        int_ip=net['net-ip'] + "." + last_octet,
                        dns="8.8.8.8"),
                    "/etc/network/interfaces.d/eth{int_num}.cfg".format(int_num=index_net),
                    "eth{int_num}".format(int_num=index_net)
                ])
            else:
                ints.append([
                    config['params']['manual_interface_template'].format(
                        int_name="eth"+str(index_net),),
                    "/etc/network/interfaces.d/eth{int_num}.cfg".format(int_num=index_net),
                    "eth{int_num}".format(int_num=index_net)
                    ])
            index_net += 1
        else:
            continue
    for interface in ints:
        user_seed_yaml["write_files"].append({
            "content": interface[0],
            "path": interface[1],
            })
    hosts_file = config['params']['hosts_template'].format(
        server_name=box["hostname"],
        domain_name=DOMAIN_NAME,
    )
    user_seed_yaml["write_files"].append({
        "content": hosts_file,
        "path": "/etc/hosts",
        })
    ## I'm really tired to investigate why this doesn't work!!!!
    #hostname_file = config['params']['hostname_template'].format(
    #    server_name=box["hostname"],
    #)
    #user_seed_yaml["write_files"].append({
    #    "content": hostname_file,
    #    "path": "/etc/hostname",
    #    })
    full_cmd = []
    for cmd in user_seed_yaml['runcmd']:
        if "hostname" in cmd:
            full_cmd.append("/bin/hostname " + box["hostname"])
            full_cmd.append("/bin/echo " + box["hostname"] + " > /etc/hostname")
        elif "ifdown" in cmd:
            for interf in ints:
                int_name = interf[2]
                full_cmd.append("/sbin/ifdown {int} && /sbin/ifup {int}".format(int=int_name))
            full_cmd.append("/etc/init.d/networking restart")
        else:
            full_cmd.append(cmd)
    user_seed_yaml['runcmd'] = full_cmd
    return yaml.dump(user_seed_yaml)


def delete_all(conn=None, lab_id=None, img_path=None, conf=None):
    if img_path:
        remove_all_imgs(img_path, lab_id)
    if conf:
        for i in [j for j in conf['params'] if "server" in j]:
            basic_name = lab_id + "-" + i.split(".")[0]
            delete_vm(conn, basic_name)
            for num in xrange(10):
                delete_vm(conn, basic_name + "%.2d" % num)<commit_msg>Fixed deleting all lab environment boxes from the node<commit_after>#!/usr/bin/python
import libvirt
import argparse
import random
import yaml
import os
import re
import sys
import logging
import subprocess

from xml.etree import ElementTree as et
from tempfile import mkstemp
from libvirt import libvirtError

__author__ = 'sshnaidm'


def rand_mac():
    mac = [0x52, 0x54, 0x00,
           random.randint(0x00, 0xff),
           random.randint(0x00, 0xff),
           random.randint(0x00, 0xff)]
    return ':'.join(["%02x" % x for x in mac])

## TODO: remove GLOBALS
name_woext = "trusty-server-cloudimg-amd64-disk1"
EXTERNAL_NET = "external"
DOMAIN_NAME = "domain.name"


def handler(ctxt, err):
    global errno
    errno = err
libvirt.registerErrorHandler(handler, 'context')


def run_cmd(cmd):
    """
        Return the exit status and output to stdout and stderr.
    """
    logging.debug("Running command: %s" % " ".join(cmd))
    proc = subprocess.Popen(cmd, stderr=subprocess.PIPE,
                            stdout=subprocess.PIPE,
                            close_fds=True)
    ret = proc.wait()
    print >> sys.stderr, "\n".join(proc.stdout.readlines()), "\n".join(proc.stderr.readlines())
    return ret, proc.stdout.readlines(), proc.stderr.readlines()


def vm_create(conn, xml):
    return conn.defineXML(xml)


def net_create(conn, xml):
    net = conn.networkDefineXML(xml)
    net.create()
    net.setAutostart(True)
    return net


def net_undefine(conn, net_name):
    try:
        net = conn.networkLookupByName(net_name)
    except libvirtError:
        print >> sys.stderr, "Net wans't found, nothing to delete"
        return
    try:
        net.destroy()
    except libvirtError:
        print >> sys.stderr, "Net wans't active, nothing to delete"
        return
    net.undefine()


def pool_delete(conn, name):
    try:
        pool = conn.storagePoolLookupByName(name)
    except libvirtError:
        print >> sys.stderr, "Pool not found, nothing to delete"
        return
    try:
        pool.destroy()
    except libvirtError:
        print >> sys.stderr, "Pool wans't active, just undefine it"
    pool.undefine()


def pool_create(conn, xml):
    pool = conn.storagePoolDefineXML(xml)
    pool.create()
    pool.setAutostart(True)
    return pool


def pool_found(conn, name):
    try:
        conn.storagePoolLookupByName(name)
        return True
    except libvirtError:
        return False


def convert_image(path, new_path):
    ret, ignore, stderr = run_cmd(["qemu-img", "convert", "-O",
                                   "qcow2", path, new_path])
    print >> sys.stderr, "\n".join(ignore), "\n".join(stderr)
    if ret == 127:
        ret, ignore, stderr = run_cmd(["kvm-img", "convert", "-O",
                                       "qcow2", path, new_path])
    if ret != 0:
        raise RuntimeError("Disk conversion failed with "
                           "exit status %d: %s" % (ret, "".join(stderr)))
    #if len(stderr):
    #    print >> sys.stderr, stderr


def main_disk_create(name="", size=0, config=None, conn=None, img_path="", boot_type="net", user_seed_yaml=None,
                     img_uncomp_path=None, lab_id="lab1"):
    path = os.path.join(img_path, name + ".qcow2")
    size_bytes = size*1024*1024*1024
    if boot_type == "net":
        vol_xml = config["params"]["vol"]["xml"].format(name=name, size=size_bytes, path=path)
        #_, tmp_file = mkstemp(prefix="disk_vol_")
        #with open(tmp_file, "w") as f:
        #    f.write(vol_xml)
        #run_cmd(["virsh", "vol-create",  "--pool", "default-" + lab_id, "--file", tmp_file])
        pool = conn.storagePoolLookupByName("default-" + lab_id)
        vol = pool.createXML(vol_xml, 1)
        #os.remove(tmp_file)
        return config["params"]["vol"]["virt_disk"].format(output_file=path)
    elif boot_type == "cloudimg":
        seed_disk = os.path.join(img_path, name + "-seed.qcow2")
        #img_uncomp_path=os.path.join(IMG_PATH, "lab-backing.qcow2")
        _, tmp_file = mkstemp(prefix="useryaml_")
        with open(tmp_file, "w") as f:
            f.write("#cloud-config\n" + user_seed_yaml)
        run_cmd(["qemu-img", "create", "-f", "qcow2", "-b", img_uncomp_path, path, "%sG" % size])
        run_cmd(["cloud-localds", seed_disk, tmp_file])
        os.remove(tmp_file)
        return config["params"]["vol"]["cloudimg_disk"].format(output_file=path, seed_disk=seed_disk)


def create_vm(conn, xml):
    vm = conn.defineXML(xml)
    vm.create()
    return vm


def findvm(conn, name):
    return [i.name() for i in conn.listAllDomains() if name in i.name()]


def delete_vm(conn, name):
    try:
        vm = conn.lookupByName(name)
    except libvirtError:
        print >> sys.stderr, "Domain {name} not found, nothing to delete".format(name=name)
        return
    try:
        vm.destroy()
    except libvirtError:
        pass
    vm.undefine()
    print >> sys.stderr, "Domain {name} deleted".format(name=name)


def remove_all_imgs(img_path, lab_id):
    if os.path.exists(img_path):
        for i in [j for j in os.listdir(img_path) if lab_id in j]:
            os.remove(os.path.join(img_path, i))


def create_seed_yaml(config, box=None):
    if not box:
        return None
    user_seed_yaml_string = config['params'][box]['user-yaml']
    user_seed_yaml = yaml.load(user_seed_yaml_string)
    user_seed_yaml['users'][1]['ssh-authorized-keys'] = config['params']['id_rsa_pub']
    user_seed_yaml['users'][1]['passwd'] = ("<PASSWORD>"
                                            "CHhs42r/bPm7ySXdYLoHCdpg3SLlWZlv9FnRAUhgp8C23DiVZr9.")
    user_seed_yaml['write_files'][0]['content'] = "\n".join(config['params']['id_rsa_pub'])
    return yaml.dump(user_seed_yaml)


def create_host_seed_yaml(config, box, btype):
    user_seed_yaml_string = config['params'][btype]['user-yaml']
    user_seed_yaml = yaml.load(user_seed_yaml_string)
    user_seed_yaml['users'][1]['ssh-authorized-keys'] = config['params']['id_rsa_pub']
    user_seed_yaml['users'][1]['passwd'] = ("<PASSWORD>=4<PASSWORD>"
                                            "<PASSWORD>.")
    user_seed_yaml['write_files'][0]['content'] = "\n".join(config['params']['id_rsa_pub'])
    ints = []
    last_octet = box['ip'].split(".")[-1]
    #sorted_nets = sorted(networks.keys(), key=lambda x: int(networks[x]["net-ip"].split(".")[2]))
    sorted_nets = config['params']['networks']
    parsed = et.fromstring(config['params'][btype]['xml'])
    used_nets = [i.attrib['network'] for i in parsed.findall(".//interface[@type='network']/source")]
    index_net = 1
    for k, net in enumerate(sorted_nets[1:]):
        if "{net_" + net['role'] + "_name}" in used_nets:
            if net['role'] != EXTERNAL_NET:
                ints.append([
                    config['params']['static_interface_template'].format(
                        int_name="eth"+str(index_net),
                        net_ip=net['net-ip'],
                        int_ip=net['net-ip'] + "." + last_octet,
                        dns="8.8.8.8"),
                    "/etc/network/interfaces.d/eth{int_num}.cfg".format(int_num=index_net),
                    "eth{int_num}".format(int_num=index_net)
                ])
            else:
                ints.append([
                    config['params']['manual_interface_template'].format(
                        int_name="eth"+str(index_net),),
                    "/etc/network/interfaces.d/eth{int_num}.cfg".format(int_num=index_net),
                    "eth{int_num}".format(int_num=index_net)
                    ])
            index_net += 1
        else:
            continue
    for interface in ints:
        user_seed_yaml["write_files"].append({
            "content": interface[0],
            "path": interface[1],
            })
    hosts_file = config['params']['hosts_template'].format(
        server_name=box["hostname"],
        domain_name=DOMAIN_NAME,
    )
    user_seed_yaml["write_files"].append({
        "content": hosts_file,
        "path": "/etc/hosts",
        })
    ## I'm really tired to investigate why this doesn't work!!!!
    #hostname_file = config['params']['hostname_template'].format(
    #    server_name=box["hostname"],
    #)
    #user_seed_yaml["write_files"].append({
    #    "content": hostname_file,
    #    "path": "/etc/hostname",
    #    })
    full_cmd = []
    for cmd in user_seed_yaml['runcmd']:
        if "hostname" in cmd:
            full_cmd.append("/bin/hostname " + box["hostname"])
            full_cmd.append("/bin/echo " + box["hostname"] + " > /etc/hostname")
        elif "ifdown" in cmd:
            for interf in ints:
                int_name = interf[2]
                full_cmd.append("/sbin/ifdown {int} && /sbin/ifup {int}".format(int=int_name))
            full_cmd.append("/etc/init.d/networking restart")
        else:
            full_cmd.append(cmd)
    user_seed_yaml['runcmd'] = full_cmd
    return yaml.dump(user_seed_yaml)


def delete_all(conn=None, lab_id=None, img_path=None, conf=None):
    if img_path:
        remove_all_imgs(img_path, lab_id)
    if conf:
        for i in [j for j in conf['params'] if "server" in j]:
            basic_name = lab_id + "-"
            vms = findvm(conn, basic_name)
            for vm_name in vms:
                delete_vm(conn, vm_name)
{'fraction_non_alphanumeric': 0.14752342704149934, 'fraction_numerical': 0.010334672021419009, 'mean_word_length': 2.57572276469462, 'pattern_counts': {'":': 16, '<': 10, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 28}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7127088', 'n_tokens_mistral': 6329, 'n_tokens_neox': 6178, 'n_words': 1174}