<commit_before>__author__ = 'frank'

import os.path
import traceback

from kvmagent import kvmagent
from kvmagent.plugins.imagestore import ImageStoreClient
from zstacklib.utils import jsonobject
from zstacklib.utils import http
from zstacklib.utils import log
from zstacklib.utils import shell
from zstacklib.utils import linux
from zstacklib.utils.bash import *
import zstacklib.utils.uuidhelper as uuidhelper

logger = log.get_logger(__name__)

class AgentResponse(object):
    def __init__(self):
        self.totalCapacity = None
        self.availableCapacity = None
        self.success = None
        self.error = None

class RevertVolumeFromSnapshotRsp(AgentResponse):
    def __init__(self):
        super(RevertVolumeFromSnapshotRsp, self).__init__()
        self.newVolumeInstallPath = None

class MergeSnapshotRsp(AgentResponse):
    def __init__(self):
        super(MergeSnapshotRsp, self).__init__()
        self.size = None
        self.actualSize = None

class RebaseAndMergeSnapshotsRsp(AgentResponse):
    def __init__(self):
        super(RebaseAndMergeSnapshotsRsp, self).__init__()
        self.size = None
        self.actualSize = None

class CheckBitsRsp(AgentResponse):
    def __init__(self):
        super(CheckBitsRsp, self).__init__()
        self.existing = False

class GetMd5Rsp(AgentResponse):
    def __init__(self):
        super(GetMd5Rsp, self).__init__()
        self.md5s = None

class GetBackingFileRsp(AgentResponse):
    def __init__(self):
        super(GetBackingFileRsp, self).__init__()
        self.size = None
        self.backingFilePath = None

class GetVolumeSizeRsp(AgentResponse):
    def __init__(self):
        super(GetVolumeSizeRsp, self).__init__()
        self.actualSize = None
        self.size = None

class GetVolumeBaseImagePathRsp(AgentResponse):
    def __init__(self):
        super(GetVolumeBaseImagePathRsp, self).__init__()
        self.path = None

class GetQCOW2ReferenceRsp(AgentResponse):
    def __init__(self):
        super(GetQCOW2ReferenceRsp, self).__init__()
        self.referencePaths = None

class LocalStoragePlugin(kvmagent.KvmAgent):

    INIT_PATH = "/localstorage/init";
    GET_PHYSICAL_CAPACITY_PATH = "/localstorage/getphysicalcapacity";
    CREATE_EMPTY_VOLUME_PATH = "/localstorage/volume/createempty";
    CREATE_VOLUME_FROM_CACHE_PATH = "/localstorage/volume/createvolumefromcache";
    DELETE_BITS_PATH = "/localstorage/delete";
    UPLOAD_BIT_PATH = "/localstorage/sftp/upload";
    DOWNLOAD_BIT_PATH = "/localstorage/sftp/download";
    UPLOAD_TO_IMAGESTORE_PATH = "/localstorage/imagestore/upload"
    COMMIT_TO_IMAGESTORE_PATH = "/localstorage/imagestore/commit"
    DOWNLOAD_FROM_IMAGESTORE_PATH = "/localstorage/imagestore/download"
    REVERT_SNAPSHOT_PATH = "/localstorage/snapshot/revert";
    MERGE_SNAPSHOT_PATH = "/localstorage/snapshot/merge";
    MERGE_AND_REBASE_SNAPSHOT_PATH = "/localstorage/snapshot/mergeandrebase";
    OFFLINE_MERGE_PATH = "/localstorage/snapshot/offlinemerge";
    CREATE_TEMPLATE_FROM_VOLUME = "/localstorage/volume/createtemplate"
    CHECK_BITS_PATH = "/localstorage/checkbits"
    REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH = "/localstorage/volume/rebaserootvolumetobackingfile"
    VERIFY_SNAPSHOT_CHAIN_PATH = "/localstorage/snapshot/verifychain"
    REBASE_SNAPSHOT_BACKING_FILES_PATH = "/localstorage/snapshot/rebasebackingfiles"
    COPY_TO_REMOTE_BITS_PATH = "/localstorage/copytoremote"
    GET_MD5_PATH = "/localstorage/getmd5"
    CHECK_MD5_PATH = "/localstorage/checkmd5"
    GET_BACKING_FILE_PATH = "/localstorage/volume/getbackingfile"
    GET_VOLUME_SIZE = "/localstorage/volume/getsize"
    GET_BASE_IMAGE_PATH = "/localstorage/volume/getbaseimagepath"
    GET_QCOW2_REFERENCE = "/localstorage/getqcow2reference"

    def start(self):
        http_server = kvmagent.get_http_server()
        http_server.register_async_uri(self.INIT_PATH, self.init)
        http_server.register_async_uri(self.GET_PHYSICAL_CAPACITY_PATH, self.get_physical_capacity)
        http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
        http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume_from_template)
        http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete)
        http_server.register_async_uri(self.DOWNLOAD_BIT_PATH, self.download_from_sftp)
        http_server.register_async_uri(self.UPLOAD_BIT_PATH, self.upload_to_sftp)
        http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
        http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
        http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
        http_server.register_async_uri(self.REVERT_SNAPSHOT_PATH, self.revert_snapshot)
        http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
        http_server.register_async_uri(self.MERGE_AND_REBASE_SNAPSHOT_PATH, self.merge_and_rebase_snapshot)
        http_server.register_async_uri(self.OFFLINE_MERGE_PATH, self.offline_merge_snapshot)
        http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME, self.create_template_from_volume)
        http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
        http_server.register_async_uri(self.REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH, self.rebase_root_volume_to_backing_file)
        http_server.register_async_uri(self.VERIFY_SNAPSHOT_CHAIN_PATH, self.verify_backing_file_chain)
        http_server.register_async_uri(self.REBASE_SNAPSHOT_BACKING_FILES_PATH, self.rebase_backing_files)
        http_server.register_async_uri(self.COPY_TO_REMOTE_BITS_PATH, self.copy_bits_to_remote)
        http_server.register_async_uri(self.GET_MD5_PATH, self.get_md5)
        http_server.register_async_uri(self.CHECK_MD5_PATH, self.check_md5)
        http_server.register_async_uri(self.GET_BACKING_FILE_PATH, self.get_backing_file_path)
        http_server.register_async_uri(self.GET_VOLUME_SIZE, self.get_volume_size)
        http_server.register_async_uri(self.GET_BASE_IMAGE_PATH, self.get_volume_base_image_path)
        http_server.register_async_uri(self.GET_QCOW2_REFERENCE, self.get_qcow2_reference)

        self.path = None
        self.imagestore_client = ImageStoreClient()

    def stop(self):
        pass

    @kvmagent.replyerror
    def get_qcow2_reference(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        out = shell.call('find %s -type f' % cmd.searchingDir)

        rsp = GetQCOW2ReferenceRsp()
        rsp.referencePaths = []
        for f in out.split('\n'):
            f = f.strip(' \t\r\n')
            if not f: continue
            backing_file = linux.qcow2_get_backing_file(f)
            if backing_file == cmd.path:
                rsp.referencePaths.append(f)

        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def get_volume_size(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = GetVolumeSizeRsp()
        rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def get_volume_base_image_path(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = GetVolumeBaseImagePathRsp()
        rsp.path = linux.get_qcow2_base_image_path_recusively(cmd.installPath)
        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def get_backing_file_path(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        out = linux.qcow2_get_backing_file(cmd.path)
        rsp = GetBackingFileRsp()

        if out:
            rsp.backingFilePath = out
            rsp.size = os.path.getsize(out)

        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def get_md5(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = GetMd5Rsp()
        rsp.md5s = []
        for to in cmd.md5s:
            md5 = shell.call("md5sum %s | cut -d ' ' -f 1" % to.path)
            rsp.md5s.append({
                'resourceUuid': to.resourceUuid,
                'path': to.path,
                'md5': md5
            })

        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def check_md5(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        for to in cmd.md5s:
            dst_md5 = shell.call("md5sum %s | cut -d ' ' -f 1" % to.path)
            if dst_md5 != to.md5:
                raise Exception("MD5 unmatch. The file[uuid:%s, path:%s]'s md5 (src host:%s, dst host:%s)" %
                                (to.resourceUuid, to.path, to.md5, dst_md5))

        rsp = AgentResponse()
        return jsonobject.dumps(rsp)


    def _get_disk_capacity(self):
        return linux.get_disk_capacity_by_df(self.path)

    @kvmagent.replyerror
    @in_bash
    def copy_bits_to_remote(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
        for path in set(chain):
            PATH = path
            PASSWORD = cmd.dstPassword
            USER = cmd.dstUsername
            IP = cmd.dstIp
            bash_errorout('rsync -a --relative {{PATH}} --rsh="/usr/bin/sshpass -p {{PASSWORD}} ssh -o StrictHostKeyChecking=no -l {{USER}}" {{IP}}:/')
            bash_errorout('/usr/bin/sshpass -p {{PASSWORD}} ssh {{USER}}@{{IP}} "/bin/sync {{PATH}}"')

        rsp = AgentResponse()
        rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def verify_backing_file_chain(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        for sp in cmd.snapshots:
            if not os.path.exists(sp.path):
                raise Exception('cannot find the file[%s]' % sp.path)

            if sp.parentPath and not os.path.exists(sp.parentPath):
                raise Exception('cannot find the backing file[%s]' % sp.parentPath)

            if sp.parentPath:
                out = linux.qcow2_get_backing_file(sp.path)

                if sp.parentPath != out:
                    raise Exception("resource[Snapshot or Volume, uuid:%s, path:%s]'s backing file[%s] is not equal to %s" %
                                (sp.snapshotUuid, sp.path, out, sp.parentPath))

        return jsonobject.dumps(AgentResponse())
<commit_msg>support non 22 port migrate  for https://github.com/zxwing/premium/issues/1420<commit_after>__author__ = 'frank'

import os.path
import traceback

from kvmagent import kvmagent
from kvmagent.plugins.imagestore import ImageStoreClient
from zstacklib.utils import jsonobject
from zstacklib.utils import http
from zstacklib.utils import log
from zstacklib.utils import shell
from zstacklib.utils import linux
from zstacklib.utils.bash import *
import zstacklib.utils.uuidhelper as uuidhelper

logger = log.get_logger(__name__)

class AgentResponse(object):
    def __init__(self):
        self.totalCapacity = None
        self.availableCapacity = None
        self.success = None
        self.error = None

class RevertVolumeFromSnapshotRsp(AgentResponse):
    def __init__(self):
        super(RevertVolumeFromSnapshotRsp, self).__init__()
        self.newVolumeInstallPath = None

class MergeSnapshotRsp(AgentResponse):
    def __init__(self):
        super(MergeSnapshotRsp, self).__init__()
        self.size = None
        self.actualSize = None

class RebaseAndMergeSnapshotsRsp(AgentResponse):
    def __init__(self):
        super(RebaseAndMergeSnapshotsRsp, self).__init__()
        self.size = None
        self.actualSize = None

class CheckBitsRsp(AgentResponse):
    def __init__(self):
        super(CheckBitsRsp, self).__init__()
        self.existing = False

class GetMd5Rsp(AgentResponse):
    def __init__(self):
        super(GetMd5Rsp, self).__init__()
        self.md5s = None

class GetBackingFileRsp(AgentResponse):
    def __init__(self):
        super(GetBackingFileRsp, self).__init__()
        self.size = None
        self.backingFilePath = None

class GetVolumeSizeRsp(AgentResponse):
    def __init__(self):
        super(GetVolumeSizeRsp, self).__init__()
        self.actualSize = None
        self.size = None

class GetVolumeBaseImagePathRsp(AgentResponse):
    def __init__(self):
        super(GetVolumeBaseImagePathRsp, self).__init__()
        self.path = None

class GetQCOW2ReferenceRsp(AgentResponse):
    def __init__(self):
        super(GetQCOW2ReferenceRsp, self).__init__()
        self.referencePaths = None

class LocalStoragePlugin(kvmagent.KvmAgent):

    INIT_PATH = "/localstorage/init";
    GET_PHYSICAL_CAPACITY_PATH = "/localstorage/getphysicalcapacity";
    CREATE_EMPTY_VOLUME_PATH = "/localstorage/volume/createempty";
    CREATE_VOLUME_FROM_CACHE_PATH = "/localstorage/volume/createvolumefromcache";
    DELETE_BITS_PATH = "/localstorage/delete";
    UPLOAD_BIT_PATH = "/localstorage/sftp/upload";
    DOWNLOAD_BIT_PATH = "/localstorage/sftp/download";
    UPLOAD_TO_IMAGESTORE_PATH = "/localstorage/imagestore/upload"
    COMMIT_TO_IMAGESTORE_PATH = "/localstorage/imagestore/commit"
    DOWNLOAD_FROM_IMAGESTORE_PATH = "/localstorage/imagestore/download"
    REVERT_SNAPSHOT_PATH = "/localstorage/snapshot/revert";
    MERGE_SNAPSHOT_PATH = "/localstorage/snapshot/merge";
    MERGE_AND_REBASE_SNAPSHOT_PATH = "/localstorage/snapshot/mergeandrebase";
    OFFLINE_MERGE_PATH = "/localstorage/snapshot/offlinemerge";
    CREATE_TEMPLATE_FROM_VOLUME = "/localstorage/volume/createtemplate"
    CHECK_BITS_PATH = "/localstorage/checkbits"
    REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH = "/localstorage/volume/rebaserootvolumetobackingfile"
    VERIFY_SNAPSHOT_CHAIN_PATH = "/localstorage/snapshot/verifychain"
    REBASE_SNAPSHOT_BACKING_FILES_PATH = "/localstorage/snapshot/rebasebackingfiles"
    COPY_TO_REMOTE_BITS_PATH = "/localstorage/copytoremote"
    GET_MD5_PATH = "/localstorage/getmd5"
    CHECK_MD5_PATH = "/localstorage/checkmd5"
    GET_BACKING_FILE_PATH = "/localstorage/volume/getbackingfile"
    GET_VOLUME_SIZE = "/localstorage/volume/getsize"
    GET_BASE_IMAGE_PATH = "/localstorage/volume/getbaseimagepath"
    GET_QCOW2_REFERENCE = "/localstorage/getqcow2reference"

    def start(self):
        http_server = kvmagent.get_http_server()
        http_server.register_async_uri(self.INIT_PATH, self.init)
        http_server.register_async_uri(self.GET_PHYSICAL_CAPACITY_PATH, self.get_physical_capacity)
        http_server.register_async_uri(self.CREATE_EMPTY_VOLUME_PATH, self.create_empty_volume)
        http_server.register_async_uri(self.CREATE_VOLUME_FROM_CACHE_PATH, self.create_root_volume_from_template)
        http_server.register_async_uri(self.DELETE_BITS_PATH, self.delete)
        http_server.register_async_uri(self.DOWNLOAD_BIT_PATH, self.download_from_sftp)
        http_server.register_async_uri(self.UPLOAD_BIT_PATH, self.upload_to_sftp)
        http_server.register_async_uri(self.UPLOAD_TO_IMAGESTORE_PATH, self.upload_to_imagestore)
        http_server.register_async_uri(self.COMMIT_TO_IMAGESTORE_PATH, self.commit_to_imagestore)
        http_server.register_async_uri(self.DOWNLOAD_FROM_IMAGESTORE_PATH, self.download_from_imagestore)
        http_server.register_async_uri(self.REVERT_SNAPSHOT_PATH, self.revert_snapshot)
        http_server.register_async_uri(self.MERGE_SNAPSHOT_PATH, self.merge_snapshot)
        http_server.register_async_uri(self.MERGE_AND_REBASE_SNAPSHOT_PATH, self.merge_and_rebase_snapshot)
        http_server.register_async_uri(self.OFFLINE_MERGE_PATH, self.offline_merge_snapshot)
        http_server.register_async_uri(self.CREATE_TEMPLATE_FROM_VOLUME, self.create_template_from_volume)
        http_server.register_async_uri(self.CHECK_BITS_PATH, self.check_bits)
        http_server.register_async_uri(self.REBASE_ROOT_VOLUME_TO_BACKING_FILE_PATH, self.rebase_root_volume_to_backing_file)
        http_server.register_async_uri(self.VERIFY_SNAPSHOT_CHAIN_PATH, self.verify_backing_file_chain)
        http_server.register_async_uri(self.REBASE_SNAPSHOT_BACKING_FILES_PATH, self.rebase_backing_files)
        http_server.register_async_uri(self.COPY_TO_REMOTE_BITS_PATH, self.copy_bits_to_remote)
        http_server.register_async_uri(self.GET_MD5_PATH, self.get_md5)
        http_server.register_async_uri(self.CHECK_MD5_PATH, self.check_md5)
        http_server.register_async_uri(self.GET_BACKING_FILE_PATH, self.get_backing_file_path)
        http_server.register_async_uri(self.GET_VOLUME_SIZE, self.get_volume_size)
        http_server.register_async_uri(self.GET_BASE_IMAGE_PATH, self.get_volume_base_image_path)
        http_server.register_async_uri(self.GET_QCOW2_REFERENCE, self.get_qcow2_reference)

        self.path = None
        self.imagestore_client = ImageStoreClient()

    def stop(self):
        pass

    @kvmagent.replyerror
    def get_qcow2_reference(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        out = shell.call('find %s -type f' % cmd.searchingDir)

        rsp = GetQCOW2ReferenceRsp()
        rsp.referencePaths = []
        for f in out.split('\n'):
            f = f.strip(' \t\r\n')
            if not f: continue
            backing_file = linux.qcow2_get_backing_file(f)
            if backing_file == cmd.path:
                rsp.referencePaths.append(f)

        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def get_volume_size(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = GetVolumeSizeRsp()
        rsp.size, rsp.actualSize = linux.qcow2_size_and_actual_size(cmd.installPath)
        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def get_volume_base_image_path(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = GetVolumeBaseImagePathRsp()
        rsp.path = linux.get_qcow2_base_image_path_recusively(cmd.installPath)
        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def get_backing_file_path(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        out = linux.qcow2_get_backing_file(cmd.path)
        rsp = GetBackingFileRsp()

        if out:
            rsp.backingFilePath = out
            rsp.size = os.path.getsize(out)

        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def get_md5(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = GetMd5Rsp()
        rsp.md5s = []
        for to in cmd.md5s:
            md5 = shell.call("md5sum %s | cut -d ' ' -f 1" % to.path)
            rsp.md5s.append({
                'resourceUuid': to.resourceUuid,
                'path': to.path,
                'md5': md5
            })

        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def check_md5(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        for to in cmd.md5s:
            dst_md5 = shell.call("md5sum %s | cut -d ' ' -f 1" % to.path)
            if dst_md5 != to.md5:
                raise Exception("MD5 unmatch. The file[uuid:%s, path:%s]'s md5 (src host:%s, dst host:%s)" %
                                (to.resourceUuid, to.path, to.md5, dst_md5))

        rsp = AgentResponse()
        return jsonobject.dumps(rsp)


    def _get_disk_capacity(self):
        return linux.get_disk_capacity_by_df(self.path)

    @kvmagent.replyerror
    @in_bash
    def copy_bits_to_remote(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
        for path in set(chain):
            PATH = path
            PASSWORD = cmd.dstPassword
            USER = cmd.dstUsername
            IP = cmd.dstIp
            PORT = (cmd.dstPort and cmd.dstPort or "22")
            bash_errorout('rsync -a --relative {{PATH}} --rsh="/usr/bin/sshpass -p {{PASSWORD}} ssh -o StrictHostKeyChecking=no -p {{PORT}} -l {{USER}}" {{IP}}:/')
            bash_errorout('/usr/bin/sshpass -p {{PASSWORD}} ssh -p {{PORT}} {{USER}}@{{IP}} "/bin/sync {{PATH}}"')

        rsp = AgentResponse()
        rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity()
        return jsonobject.dumps(rsp)

    @kvmagent.replyerror
    def verify_backing_file_chain(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        for sp in cmd.snapshots:
            if not os.path.exists(sp.path):
                raise Exception('cannot find the file[%s]' % sp.path)

            if sp.parentPath and not os.path.exists(sp.parentPath):
                raise Exception('cannot find the backing file[%s]' % sp.parentPath)

            if sp.parentPath:
                out = linux.qcow2_get_backing_file(sp.path)

                if sp.parentPath != out:
                    raise Exception("resource[Snapshot or Volume, uuid:%s, path:%s]'s backing file[%s] is not equal to %s" %
                                (sp.snapshotUuid, sp.path, out, sp.parentPath))

        return jsonobject.dumps(AgentResponse())

{'fraction_non_alphanumeric': 0.09512727963525835, 'fraction_numerical': 0.004749240121580547, 'mean_word_length': 3.325595727198028, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4115237', 'n_tokens_mistral': 7145, 'n_tokens_neox': 7027, 'n_words': 1040}