<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import json
import commands

from kvmagent import kvmagent
from zstacklib.utils import jsonobject
from zstacklib.utils import linux
from zstacklib.utils import log
from zstacklib.utils import shell
from zstacklib.utils import http
from zstacklib.utils import thread
from zstacklib.utils.bash import in_bash
from zstacklib.utils.linux import shellquote
from zstacklib.utils.plugin import completetask

logger = log.get_logger(__name__)


class AgentRsp(object):
    def __init__(self):
        self.success = True
        self.error = None


class CheckBitsRsp(AgentRsp):
    def __init__(self):
        super(CheckBitsRsp, self).__init__()
        self.existing = False


class ConvertRsp(AgentRsp):
    def __init__(self):
        super(ConvertRsp, self).__init__()
        self.rootVolumeInfo = None
        self.dataVolumeInfos = []
        self.bootMode = None

QOS_IFB = "ifb0"

class VMwareV2VPlugin(kvmagent.KvmAgent):
    INIT_PATH = "/vmwarev2v/conversionhost/init"
    CONVERT_PATH = "/vmwarev2v/conversionhost/convert"
    CLEAN_PATH = "/vmwarev2v/conversionhost/clean"
    CHECK_BITS = "/vmwarev2v/conversionhost/checkbits"
    CONFIG_QOS_PATH = "/vmwarev2v/conversionhost/qos/config"
    DELETE_QOS_PATH = "/vmwarev2v/conversionhost/qos/delete"
    CANCEL_CONVERT_PATH = "/vmwarev2v/conversionhost/convert/cancel"

    def start(self):
        http_server = kvmagent.get_http_server()
        http_server.register_async_uri(self.INIT_PATH, self.init)
        http_server.register_async_uri(self.CONVERT_PATH, self.convert)
        http_server.register_async_uri(self.CLEAN_PATH, self.clean)
        http_server.register_async_uri(self.CANCEL_CONVERT_PATH, self.clean_convert)
        http_server.register_async_uri(self.CHECK_BITS, self.check_bits)
        http_server.register_async_uri(self.CONFIG_QOS_PATH, self.config_qos)
        http_server.register_async_uri(self.DELETE_QOS_PATH, self.delete_qos)

    def stop(self):
        pass

    @in_bash
    @kvmagent.replyerror
    def init(self, req):
        # TODO do not use docker
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = AgentRsp()

        cmdstr = 'mkdir -p /var/lib/zstack/v2v && cd /var/lib/zstack/v2v && wget -c {} -O zstack-windows-virtio-driver.iso'.format(
            cmd.virtioDriverUrl)
        if shell.run(cmdstr) != 0:
            rsp.success = False
            rsp.error = "failed to download zstack-windows-virtio-driver.iso from management node to v2v conversion host"
            return jsonobject.dumps(rsp)

        return jsonobject.dumps(rsp)

    @in_bash
    @kvmagent.replyerror
    @completetask
    def convert(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = ConvertRsp()

        storage_dir = os.path.join(cmd.storagePath, cmd.srcVmUuid)

        def validate_and_make_dir(_dir):
            existing = os.path.exists(_dir)
            if not existing:
                shell.call("mkdir -p %s" % _dir)
            return existing<commit_msg>Add update conversion host msg<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import json
import commands

from kvmagent import kvmagent
from zstacklib.utils import jsonobject
from zstacklib.utils import linux
from zstacklib.utils import log
from zstacklib.utils import shell
from zstacklib.utils import http
from zstacklib.utils import thread
from zstacklib.utils.bash import in_bash
from zstacklib.utils.linux import shellquote
from zstacklib.utils.plugin import completetask

logger = log.get_logger(__name__)


class AgentRsp(object):
    def __init__(self):
        self.success = True
        self.error = None


class CheckBitsRsp(AgentRsp):
    def __init__(self):
        super(CheckBitsRsp, self).__init__()
        self.existing = False


class ConvertRsp(AgentRsp):
    def __init__(self):
        super(ConvertRsp, self).__init__()
        self.rootVolumeInfo = None
        self.dataVolumeInfos = []
        self.bootMode = None


QOS_IFB = "ifb0"

VDDK_VERSION = '/var/lib/zstack/v2v/vddk_version'
WINDOWS_VIRTIO_DRIVE_ISO_VERSION = '/var/lib/zstack/v2v/windows_virtio_version'
V2V_LIB_PATH = '/var/lib/zstack/v2v/'

class VMwareV2VPlugin(kvmagent.KvmAgent):
    INIT_PATH = "/vmwarev2v/conversionhost/init"
    CONVERT_PATH = "/vmwarev2v/conversionhost/convert"
    CLEAN_PATH = "/vmwarev2v/conversionhost/clean"
    CHECK_BITS = "/vmwarev2v/conversionhost/checkbits"
    CONFIG_QOS_PATH = "/vmwarev2v/conversionhost/qos/config"
    DELETE_QOS_PATH = "/vmwarev2v/conversionhost/qos/delete"
    CANCEL_CONVERT_PATH = "/vmwarev2v/conversionhost/convert/cancel"

    def start(self):
        http_server = kvmagent.get_http_server()
        http_server.register_async_uri(self.INIT_PATH, self.init)
        http_server.register_async_uri(self.CONVERT_PATH, self.convert)
        http_server.register_async_uri(self.CLEAN_PATH, self.clean)
        http_server.register_async_uri(self.CANCEL_CONVERT_PATH, self.clean_convert)
        http_server.register_async_uri(self.CHECK_BITS, self.check_bits)
        http_server.register_async_uri(self.CONFIG_QOS_PATH, self.config_qos)
        http_server.register_async_uri(self.DELETE_QOS_PATH, self.delete_qos)

    def stop(self):
        pass

    @in_bash
    @kvmagent.replyerror
    def init(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = AgentRsp()

        path = "/var/lib/zstack/v2v"
        if not os.path.exists(path):
            os.makedirs(path, 0775)

        if not os.path.exists(WINDOWS_VIRTIO_DRIVE_ISO_VERSION) \
                and os.path.exists(V2V_LIB_PATH + 'zstack-windows-virtio-driver.iso'):
            last_modified = shell.call("curl -I %s | grep 'Last-Modified'" % cmd.virtioDriverUrl)
            with open(WINDOWS_VIRTIO_DRIVE_ISO_VERSION, 'w') as fd:
                fd.write(last_modified.strip('\n\r'))
        else:
            last_modified = shell.call("curl -I %s | grep 'Last-Modified'" % cmd.virtioDriverUrl).strip('\n\r')

            with open(WINDOWS_VIRTIO_DRIVE_ISO_VERSION, 'r') as fd:
                version = fd.readline()

            if version != last_modified:
                cmdstr = 'cd /var/lib/zstack/v2v && wget -c {} -O zstack-windows-virtio-driver.iso'.format(
                    cmd.virtioDriverUrl)
                if shell.run(cmdstr) != 0:
                    rsp.success = False
                    rsp.error = "failed to download zstack-windows-virtio-driver.iso " \
                                "from management node to v2v conversion host"
                    return jsonobject.dumps(rsp)

                with open(WINDOWS_VIRTIO_DRIVE_ISO_VERSION, 'w') as fd:
                    fd.write(last_modified)

        if not os.path.exists(VDDK_VERSION) and os.path.exists(V2V_LIB_PATH + 'vmware-vix-disklib-distrib.tar.gz'):
            with open(VDDK_VERSION, 'w') as fd:
                fd.write(cmd.vddkLibUrl.split('/')[-1])
        else:
            current_version = cmd.vddkLibUrl.split('/')[-1]

            with open(VDDK_VERSION, 'r') as fd:
                version = fd.readline()

            if current_version != version:
                cmdstr = 'cd /var/lib/zstack/v2v && wget -c {} -O vmware-vix-disklib-distrib.tar.gz && ' \
                         'tar zxf vmware-vix-disklib-distrib.tar.gz'.format(
                    cmd.vddkLibUrl)
                if shell.run(cmdstr) != 0:
                    rsp.success = False
                    rsp.error = "failed to download vmware-vix-disklib-distrib.tar.gz " \
                                "from management node to v2v conversion host"
                    return jsonobject.dumps(rsp)

                with open(VDDK_VERSION, 'w') as fd:
                    fd.write(current_version)

        return jsonobject.dumps(rsp)

    @in_bash
    @kvmagent.replyerror
    @completetask
    def convert(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = ConvertRsp()

        storage_dir = os.path.join(cmd.storagePath, cmd.srcVmUuid)

        def validate_and_make_dir(_dir):
            existing = os.path.exists(_dir)
            if not existing:
                shell.call("mkdir -p %s" % _dir)
            return existing
{'fraction_non_alphanumeric': 0.10086977530804542, 'fraction_numerical': 0.005194491423049045, 'mean_word_length': 2.585534863577306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5992460', 'n_tokens_mistral': 2795, 'n_tokens_neox': 2688, 'n_words': 492}