<commit_before>import functools
import random
import os.path
import time

from zstacklib.utils import shell
from zstacklib.utils import bash
from zstacklib.utils import log
from zstacklib.utils import linux

logger = log.get_logger(__name__)
LV_RESERVED_SIZE = 1024*1024*4
LVM_CONFIG_PATH = "/etc/lvm"
SANLOCK_CONFIG_FILE_PATH = "/etc/sanlock/sanlock.conf"
SANLOCK_IO_TIMEOUT = 40
LVMLOCKD_LOG_FILE_PATH = "/var/log/lvmlockd/lvmlockd.log"
LVMLOCKD_LOG_LOGROTATE_PATH = "/etc/logrotate.d/lvmlockd"
LVM_CONFIG_BACKUP_PATH = "/etc/lvm/zstack-backup"
SUPER_BLOCK_BACKUP = "superblock.bak"
COMMON_TAG = "zs::sharedblock"
VOLUME_TAG = COMMON_TAG + "::volume"
IMAGE_TAG = COMMON_TAG + "::image"
ENABLE_DUP_GLOBAL_CHECK = False
thinProvisioningInitializeSize = "thinProvisioningInitializeSize"


class VolumeProvisioningStrategy(object):
    ThinProvisioning = "ThinProvisioning"
    ThickProvisioning = "ThickProvisioning"


class VmStruct(object):
    def __init__(self):
        super(VmStruct, self).__init__()
        self.pid = ""
        self.cmdline = ""
        self.root_volume = ""
        self.uuid = ""
        self.volumes = []


class LvmlockdLockType(object):
    NULL = 0
    SHARE = 1
    EXCLUSIVE = 2

    @staticmethod
    def from_abbr(abbr):
        if abbr.strip() == "sh":
            return LvmlockdLockType.SHARE
        elif abbr.strip() == "ex":
            return LvmlockdLockType.EXCLUSIVE
        elif abbr.strip() == "un":
            return LvmlockdLockType.NULL
        elif abbr.strip() == "":
            logger.warn("can not get correct lvm lock type! use null as a safe choice")
            return LvmlockdLockType.NULL
        else:
            raise Exception("unknown lock type from abbr: %s" % abbr)

    @staticmethod
    def from_str(string):
        if string == "NULL":
            return LvmlockdLockType.SHARE
        elif string == "SHARE":
            return LvmlockdLockType.EXCLUSIVE
        elif string == "EXCLUSIVE":
            return LvmlockdLockType.NULL
        else:
            raise Exception("unknown lock type from str: %s" % string)


class RetryException(Exception):
    pass


class SharedBlockCandidateStruct:
    wwids = []  # type: list[str]
    vendor = None  # type: str
    model = None  # type: str
    wwn = None  # type: str
    serial = None  # type: str
    hctl = None  # type: str
    type = None  # type: str
    size = None  # type: long
    path = None  # type: str

    def __init__(self):
        pass


def get_block_devices():
    # 1. get multi path devices
    # 2. get multi path device information from raw device
    # 3. get information of other devices
    mpath_devices = []
    block_devices = []  # type: List[SharedBlockCandidateStruct]
    cmd = shell.ShellCmd("multipath -l -v1")
    cmd(is_exception=False)
    if cmd.return_code == 0 and cmd.stdout.strip() != "":
        mpath_devices = cmd.stdout.strip().split("\n")

    for mpath_device in mpath_devices:
        try:
            cmd = shell.ShellCmd("realpath /dev/mapper/%s | grep -E -o 'dm-.*'" % mpath_device)
            cmd(is_exception=False)
            if cmd.return_code != 0 or cmd.stdout.strip() == "":
                continue

            dm = cmd.stdout.strip()
            slaves = shell.call("ls /sys/class/block/%s/slaves/" % dm).strip().split("\n")
            if slaves is None or len(slaves) == 0:
                struct = SharedBlockCandidateStruct()
                cmd = shell.ShellCmd("multipath -l /dev/mapper/%s | grep %s | grep -o ' (.*) '" % (
                    mpath_device, mpath_device))
                cmd(is_exception=True)
                struct.wwids = [cmd.stdout.strip().strip("()")]
                struct.type = "mpath"
                block_devices.append(struct)
                continue

            struct = get_device_info(slaves[0])
            cmd = shell.ShellCmd("multipath -l /dev/mapper/%s | grep %s | grep -o ' (.*) '" % (
                mpath_device, mpath_device))
            cmd(is_exception=True)
            struct.wwids = [cmd.stdout.strip().strip("()")]
            struct.type = "mpath"
            block_devices.append(struct)
        except Exception as e:
            logger.warn(linux.get_exception_stacktrace())
            continue

    disks = shell.call("lsblk -p -o NAME,TYPE | grep disk | awk '{print $1}'").strip().split()
    for disk in disks:
        try:
            if is_slave_of_multipath(disk):
                continue
            d = get_device_info(disk.strip().split("/")[-1])
            if len(d.wwids) is 0:
                continue
            if get_pv_uuid_by_path("/dev/disk/by-id/%s" % d.wwids[0]) not in ("", None):
                d.type = "lvm-pv"
            block_devices.append(d)
        except Exception as e:
            logger.warn(linux.get_exception_stacktrace())
            continue

    return block_devices


@bash.in_bash
def is_multipath_running():
    r = bash.bash_r("multipath -t")
    if r != 0:
        return False<commit_msg>Fix get sharedblock devices without friendly name<commit_after>import functools
import random
import os.path
import time

from zstacklib.utils import shell
from zstacklib.utils import bash
from zstacklib.utils import log
from zstacklib.utils import linux

logger = log.get_logger(__name__)
LV_RESERVED_SIZE = 1024*1024*4
LVM_CONFIG_PATH = "/etc/lvm"
SANLOCK_CONFIG_FILE_PATH = "/etc/sanlock/sanlock.conf"
SANLOCK_IO_TIMEOUT = 40
LVMLOCKD_LOG_FILE_PATH = "/var/log/lvmlockd/lvmlockd.log"
LVMLOCKD_LOG_LOGROTATE_PATH = "/etc/logrotate.d/lvmlockd"
LVM_CONFIG_BACKUP_PATH = "/etc/lvm/zstack-backup"
SUPER_BLOCK_BACKUP = "superblock.bak"
COMMON_TAG = "zs::sharedblock"
VOLUME_TAG = COMMON_TAG + "::volume"
IMAGE_TAG = COMMON_TAG + "::image"
ENABLE_DUP_GLOBAL_CHECK = False
thinProvisioningInitializeSize = "thinProvisioningInitializeSize"


class VolumeProvisioningStrategy(object):
    ThinProvisioning = "ThinProvisioning"
    ThickProvisioning = "ThickProvisioning"


class VmStruct(object):
    def __init__(self):
        super(VmStruct, self).__init__()
        self.pid = ""
        self.cmdline = ""
        self.root_volume = ""
        self.uuid = ""
        self.volumes = []


class LvmlockdLockType(object):
    NULL = 0
    SHARE = 1
    EXCLUSIVE = 2

    @staticmethod
    def from_abbr(abbr):
        if abbr.strip() == "sh":
            return LvmlockdLockType.SHARE
        elif abbr.strip() == "ex":
            return LvmlockdLockType.EXCLUSIVE
        elif abbr.strip() == "un":
            return LvmlockdLockType.NULL
        elif abbr.strip() == "":
            logger.warn("can not get correct lvm lock type! use null as a safe choice")
            return LvmlockdLockType.NULL
        else:
            raise Exception("unknown lock type from abbr: %s" % abbr)

    @staticmethod
    def from_str(string):
        if string == "NULL":
            return LvmlockdLockType.SHARE
        elif string == "SHARE":
            return LvmlockdLockType.EXCLUSIVE
        elif string == "EXCLUSIVE":
            return LvmlockdLockType.NULL
        else:
            raise Exception("unknown lock type from str: %s" % string)


class RetryException(Exception):
    pass


class SharedBlockCandidateStruct:
    wwids = []  # type: list[str]
    vendor = None  # type: str
    model = None  # type: str
    wwn = None  # type: str
    serial = None  # type: str
    hctl = None  # type: str
    type = None  # type: str
    size = None  # type: long
    path = None  # type: str

    def __init__(self):
        pass


def get_block_devices():
    # 1. get multi path devices
    # 2. get multi path device information from raw device
    # 3. get information of other devices
    mpath_devices = []
    block_devices = []  # type: List[SharedBlockCandidateStruct]
    cmd = shell.ShellCmd("multipath -l -v1")
    cmd(is_exception=False)
    if cmd.return_code == 0 and cmd.stdout.strip() != "":
        mpath_devices = cmd.stdout.strip().split("\n")

    for mpath_device in mpath_devices:  # type: str
        try:
            cmd = shell.ShellCmd("realpath /dev/mapper/%s | grep -E -o 'dm-.*'" % mpath_device)
            cmd(is_exception=False)
            if cmd.return_code != 0 or cmd.stdout.strip() == "":
                continue

            dm = cmd.stdout.strip()
            slaves = shell.call("ls /sys/class/block/%s/slaves/" % dm).strip().split("\n")
            if slaves is None or len(slaves) == 0:
                struct = SharedBlockCandidateStruct()
                cmd = shell.ShellCmd("udevadm info -n %s | grep dm-uuid-mpath | grep -o 'dm-uuid-mpath-\S*' | head -n 1 | awk -F '-' '{print $NF}'" % dm)
                cmd(is_exception=True)
                struct.wwids = [cmd.stdout.strip().strip("()")]
                struct.type = "mpath"
                block_devices.append(struct)
                continue

            struct = get_device_info(slaves[0])
            cmd = shell.ShellCmd("udevadm info -n %s | grep dm-uuid-mpath | grep -o 'dm-uuid-mpath-\S*' | head -n 1 | awk -F '-' '{print $NF}'" % dm)
            cmd(is_exception=True)
            struct.wwids = [cmd.stdout.strip().strip("()")]
            struct.type = "mpath"
            block_devices.append(struct)
        except Exception as e:
            logger.warn(linux.get_exception_stacktrace())
            continue

    disks = shell.call("lsblk -p -o NAME,TYPE | grep disk | awk '{print $1}'").strip().split()
    for disk in disks:
        try:
            if is_slave_of_multipath(disk):
                continue
            d = get_device_info(disk.strip().split("/")[-1])
            if len(d.wwids) is 0:
                continue
            if get_pv_uuid_by_path("/dev/disk/by-id/%s" % d.wwids[0]) not in ("", None):
                d.type = "lvm-pv"
            block_devices.append(d)
        except Exception as e:
            logger.warn(linux.get_exception_stacktrace())
            continue

    return block_devices


@bash.in_bash
def is_multipath_running():
    r = bash.bash_r("multipath -t")
    if r != 0:
        return False
{'fraction_non_alphanumeric': 0.11658211479160449, 'fraction_numerical': 0.0055704764746841735, 'mean_word_length': 2.267468313292168, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2469626', 'n_tokens_mistral': 3319, 'n_tokens_neox': 3200, 'n_words': 743}