<commit_before># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import with_statement

import re
import os
import shlex
import socket
import time
import platform
import subprocess
import mimetypes
import paramiko
import atexit
import logging
import netaddr

from tempfile import NamedTemporaryFile
from os.path import join as pjoin
from collections import defaultdict

try:
    from lxml import etree as ET
except ImportError:
    from xml.etree import ElementTree as ET

from libcloud.compute.base import NodeDriver, Node, NodeImage, NodeSize
from libcloud.compute.base import NodeState
from libcloud.compute.types import Provider
from libcloud.utils.networking import is_public_subnet
from libcloud.utils.networking import is_valid_ip_address
from libcloud.utils.py3 import basestring

try:
    import libvirt
    have_libvirt = True
except ImportError:
    raise RuntimeError('Missing "libvirt" dependency. You can install it using '
                       'pip. For example ./bin/pip install libvirt-python')


log = logging.getLogger('libcloud.compute.drivers.libvirt')


ALLOW_LIBVIRT_LOCALHOST = False
IMAGES_LOCATION = "/var/lib/libvirt/images"

# directory to store cloudinit related files etc
LIBCLOUD_DIRECTORY = "/var/lib/libvirt/libcloud"

# disk image types to create VMs from
DISK_IMAGE_TYPES = ('.img', '.raw', '.qcow', '.qcow2')


class LibvirtNodeDriver(NodeDriver):
    """
    Libvirt (http://libvirt.org/) node driver.

    To enable debug mode, set LIBVIR_DEBUG environment variable.
    """

    type = Provider.LIBVIRT
    name = 'Libvirt'
    website = 'http://libvirt.org/'

    NODE_STATE_MAP = {
        0: NodeState.TERMINATED,  # no state
        1: NodeState.RUNNING,  # domain is running
        2: NodeState.PENDING,  # domain is blocked on resource
        3: NodeState.SUSPENDED,  # domain is paused by user
        4: NodeState.TERMINATED,  # domain is being shut down
        5: NodeState.TERMINATED,  # domain is shut off
        6: NodeState.UNKNOWN,  # domain is crashed
        7: NodeState.UNKNOWN,  # domain is suspended by guest power management
    }

    def __init__(self, host, user='root', ssh_key=None,
                 ssh_port=22, tcp_port=5000, hypervisor=None):
        """
        Supports three ways to connect: local system, qemu+tcp, qemu+ssh
        :param host: IP address or hostname to connect to (usually the
        address of the KVM hypervisor)
        :param hypervisor: the IP address of the KVM hypervisor. Useful in case
        `host` has been substituted by a middleware
        :param user: the username to connect to the KVM hypervisor as
        :param ssh_key: a filename with the private key
        :param ssh_port: the SSH port to connect to when qemu+ssh is chosen
        :param tcp_port: the TCP port to connect to in case of qemu+tcp
        :return:
        """
        self._ssh_conn = None

        self.temp_key = None
        self.secret = None
        if host in ['localhost', '127.0.0.1', '0.0.0.0']:
            # local connection
            if ALLOW_LIBVIRT_LOCALHOST:
                uri = 'qemu:///system'
            else:
                raise Exception("In order to connect to local libvirt enable "
                                "ALLOW_LIBVIRT_LOCALHOST variable")
        else:
            if ssh_key:
                # if ssh key is string create temp file
                if not os.path.isfile(ssh_key):
                    key_temp_file = NamedTemporaryFile(delete=False)
                    key_temp_file.write(ssh_key.encode())
                    key_temp_file.close()
                    self.secret = key_temp_file.name
                    self.temp_key = self.secret
                else:
                    self.secret = ssh_key
                # ssh connection
                # initially attempt to connect to host/port and raise
                # exception on failure
                try:
                    socket.setdefaulttimeout(15)
                    so = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                    so.connect((host, ssh_port))
                    so.close()
                except:
                    raise Exception("Make sure host is accessible and ssh port is open")

                uri = 'qemu+ssh://%s@%s:%s/system?keyfile=%s&no_tty=1&no_verify=1' % (user, host, ssh_port, self.secret)
            else:
                # tcp connection
                try:
                    socket.setdefaulttimeout(15)
                    so = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                    so.connect((host, tcp_port))
                    so.close()
                except:
                    raise Exception("If you don't specify an ssh key, libvirt "
                                    "will try to connect to port 5000 through "
                                    "qemu+tcp")

                uri = 'qemu+tcp://%s:%s/system' % (host, tcp_port)

        self._uri = uri
        self.host = host
        self.hypervisor = hypervisor if hypervisor else host
        self.ssh_port = ssh_port
        self.key = user

        try:
            self.connection = libvirt.open(uri)
        except Exception as exc:
            if 'Could not resolve' in exc.message:
                raise Exception("Make sure hostname is accessible")
            if 'Connection refused' in exc.message:
                raise Exception("Make sure hostname is accessible and libvirt "
                                "is running")
            if 'Permission denied' in exc.message:
                raise Exception("Make sure ssh key and username are valid")
            if 'End of file while reading data' in exc.message:
                raise Exception("Make sure libvirt is running and user %s is "
                                "authorised to connect" % user)
            raise Exception("Connection error")

            atexit.register(self.disconnect)

    def list_nodes(self, show_hypervisor=True):

        # active domains
        domain_ids = self.connection.listDomainsID()
        domains = [self.connection.lookupByID(id) for id in domain_ids]
        # non active domains
        inactive_domains = map(self.connection.lookupByName,
                               self.connection.listDefinedDomains())
        domains.extend(inactive_domains)

        # get the arp table of the hypervisor. Try to connect with provided
        # ssh key and paramiko<commit_msg>Fix Python3 compatibility for exceptions in libvirt_driver<commit_after># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import with_statement

import re
import os
import shlex
import socket
import time
import platform
import subprocess
import mimetypes
import paramiko
import atexit
import logging
import netaddr

from tempfile import NamedTemporaryFile
from os.path import join as pjoin
from collections import defaultdict

try:
    from lxml import etree as ET
except ImportError:
    from xml.etree import ElementTree as ET

from libcloud.compute.base import NodeDriver, Node, NodeImage, NodeSize
from libcloud.compute.base import NodeState
from libcloud.compute.types import Provider
from libcloud.utils.networking import is_public_subnet
from libcloud.utils.networking import is_valid_ip_address
from libcloud.utils.py3 import basestring

try:
    import libvirt
    have_libvirt = True
except ImportError:
    raise RuntimeError('Missing "libvirt" dependency. You can install it using '
                       'pip. For example ./bin/pip install libvirt-python')


log = logging.getLogger('libcloud.compute.drivers.libvirt')


ALLOW_LIBVIRT_LOCALHOST = False
IMAGES_LOCATION = "/var/lib/libvirt/images"

# directory to store cloudinit related files etc
LIBCLOUD_DIRECTORY = "/var/lib/libvirt/libcloud"

# disk image types to create VMs from
DISK_IMAGE_TYPES = ('.img', '.raw', '.qcow', '.qcow2')


class LibvirtNodeDriver(NodeDriver):
    """
    Libvirt (http://libvirt.org/) node driver.

    To enable debug mode, set LIBVIR_DEBUG environment variable.
    """

    type = Provider.LIBVIRT
    name = 'Libvirt'
    website = 'http://libvirt.org/'

    NODE_STATE_MAP = {
        0: NodeState.TERMINATED,  # no state
        1: NodeState.RUNNING,  # domain is running
        2: NodeState.PENDING,  # domain is blocked on resource
        3: NodeState.SUSPENDED,  # domain is paused by user
        4: NodeState.TERMINATED,  # domain is being shut down
        5: NodeState.TERMINATED,  # domain is shut off
        6: NodeState.UNKNOWN,  # domain is crashed
        7: NodeState.UNKNOWN,  # domain is suspended by guest power management
    }

    def __init__(self, host, user='root', ssh_key=None,
                 ssh_port=22, tcp_port=5000, hypervisor=None):
        """
        Supports three ways to connect: local system, qemu+tcp, qemu+ssh
        :param host: IP address or hostname to connect to (usually the
        address of the KVM hypervisor)
        :param hypervisor: the IP address of the KVM hypervisor. Useful in case
        `host` has been substituted by a middleware
        :param user: the username to connect to the KVM hypervisor as
        :param ssh_key: a filename with the private key
        :param ssh_port: the SSH port to connect to when qemu+ssh is chosen
        :param tcp_port: the TCP port to connect to in case of qemu+tcp
        :return:
        """
        self._ssh_conn = None

        self.temp_key = None
        self.secret = None
        if host in ['localhost', '127.0.0.1', '0.0.0.0']:
            # local connection
            if ALLOW_LIBVIRT_LOCALHOST:
                uri = 'qemu:///system'
            else:
                raise Exception("In order to connect to local libvirt enable "
                                "ALLOW_LIBVIRT_LOCALHOST variable")
        else:
            if ssh_key:
                # if ssh key is string create temp file
                if not os.path.isfile(ssh_key):
                    key_temp_file = NamedTemporaryFile(delete=False)
                    key_temp_file.write(ssh_key.encode())
                    key_temp_file.close()
                    self.secret = key_temp_file.name
                    self.temp_key = self.secret
                else:
                    self.secret = ssh_key
                # ssh connection
                # initially attempt to connect to host/port and raise
                # exception on failure
                try:
                    socket.setdefaulttimeout(15)
                    so = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                    so.connect((host, ssh_port))
                    so.close()
                except:
                    raise Exception("Make sure host is accessible and ssh port is open")

                uri = 'qemu+ssh://%s@%s:%s/system?keyfile=%s&no_tty=1&no_verify=1' % (user, host, ssh_port, self.secret)
            else:
                # tcp connection
                try:
                    socket.setdefaulttimeout(15)
                    so = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                    so.connect((host, tcp_port))
                    so.close()
                except:
                    raise Exception("If you don't specify an ssh key, libvirt "
                                    "will try to connect to port 5000 through "
                                    "qemu+tcp")

                uri = 'qemu+tcp://%s:%s/system' % (host, tcp_port)

        self._uri = uri
        self.host = host
        self.hypervisor = hypervisor if hypervisor else host
        self.ssh_port = ssh_port
        self.key = user

        try:
            self.connection = libvirt.open(uri)
        except Exception as exc:
            if 'Could not resolve' in str(exc):
                raise Exception("Make sure hostname is accessible")
            if 'Connection refused' in str(exc):
                raise Exception("Make sure hostname is accessible and libvirt "
                                "is running")
            if 'Permission denied' in str(exc):
                raise Exception("Make sure ssh key and username are valid")
            if 'End of file while reading data' in str(exc):
                raise Exception("Make sure libvirt is running and user %s is "
                                "authorised to connect" % user)
            raise Exception("Connection error")

            atexit.register(self.disconnect)

    def list_nodes(self, show_hypervisor=True):

        # active domains
        domain_ids = self.connection.listDomainsID()
        domains = [self.connection.lookupByID(id) for id in domain_ids]
        # non active domains
        inactive_domains = map(self.connection.lookupByName,
                               self.connection.listDefinedDomains())
        domains.extend(inactive_domains)

        # get the arp table of the hypervisor. Try to connect with provided
        # ssh key and paramiko
{'fraction_non_alphanumeric': 0.0714732670513801, 'fraction_numerical': 0.00563164847389279, 'mean_word_length': 2.111399524118538, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3858683', 'n_tokens_mistral': 3927, 'n_tokens_neox': 3710, 'n_words': 1337}