<commit_before>package org.ovirt.engine.core.vdsbroker.irsbroker;

import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang.StringUtils;
import org.ovirt.engine.core.common.AuditLogType;
import org.ovirt.engine.core.common.FeatureSupported;
import org.ovirt.engine.core.common.businessentities.AsyncTaskStatus;
import org.ovirt.engine.core.common.businessentities.AsyncTaskStatusEnum;
import org.ovirt.engine.core.common.businessentities.BusinessEntitiesDefinitions;
import org.ovirt.engine.core.common.businessentities.Entities;
import org.ovirt.engine.core.common.businessentities.NonOperationalReason;
import org.ovirt.engine.core.common.businessentities.SpmStatus;
import org.ovirt.engine.core.common.businessentities.SpmStatusResult;
import org.ovirt.engine.core.common.businessentities.StorageDomain;
import org.ovirt.engine.core.common.businessentities.StorageDomainStatic;
import org.ovirt.engine.core.common.businessentities.StorageDomainStatus;
import org.ovirt.engine.core.common.businessentities.StorageDomainType;
import org.ovirt.engine.core.common.businessentities.StoragePool;
import org.ovirt.engine.core.common.businessentities.StoragePoolIsoMap;
import org.ovirt.engine.core.common.businessentities.StoragePoolIsoMapId;
import org.ovirt.engine.core.common.businessentities.StoragePoolStatus;
import org.ovirt.engine.core.common.businessentities.VDS;
import org.ovirt.engine.core.common.businessentities.VDSDomainsData;
import org.ovirt.engine.core.common.businessentities.VDSStatus;
import org.ovirt.engine.core.common.businessentities.VdsProtocol;
import org.ovirt.engine.core.common.businessentities.vds_spm_id_map;
import org.ovirt.engine.core.common.config.Config;
import org.ovirt.engine.core.common.config.ConfigValues;
import org.ovirt.engine.core.common.errors.EngineError;
import org.ovirt.engine.core.common.errors.EngineMessage;
import org.ovirt.engine.core.common.eventqueue.Event;
import org.ovirt.engine.core.common.eventqueue.EventQueue;
import org.ovirt.engine.core.common.eventqueue.EventResult;
import org.ovirt.engine.core.common.eventqueue.EventType;
import org.ovirt.engine.core.common.locks.LockingGroup;
import org.ovirt.engine.core.common.utils.Pair;
import org.ovirt.engine.core.common.vdscommands.ConnectStoragePoolVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.DisconnectStoragePoolVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.GetStoragePoolInfoVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStartVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStatusVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStopVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.VDSCommandType;
import org.ovirt.engine.core.common.vdscommands.VDSReturnValue;
import org.ovirt.engine.core.common.vdscommands.VdsIdVDSCommandParametersBase;
import org.ovirt.engine.core.compat.Guid;
import org.ovirt.engine.core.compat.KeyValuePairCompat;
import org.ovirt.engine.core.compat.RefObject;
import org.ovirt.engine.core.compat.TransactionScopeOption;
import org.ovirt.engine.core.compat.Version;
import org.ovirt.engine.core.dal.dbbroker.DbFacade;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase;
import org.ovirt.engine.core.di.Injector;
import org.ovirt.engine.core.utils.lock.EngineLock;
import org.ovirt.engine.core.utils.lock.LockManagerFactory;
import org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil;
import org.ovirt.engine.core.utils.timer.OnTimerMethodAnnotation;
import org.ovirt.engine.core.utils.timer.SchedulerUtil;
import org.ovirt.engine.core.utils.timer.SchedulerUtilQuartzImpl;
import org.ovirt.engine.core.utils.transaction.TransactionMethod;
import org.ovirt.engine.core.utils.transaction.TransactionSupport;
import org.ovirt.engine.core.vdsbroker.ResourceManager;
import org.ovirt.engine.core.vdsbroker.TransportFactory;
import org.ovirt.engine.core.vdsbroker.jsonrpc.JsonRpcIIrsServer;
import org.ovirt.engine.core.vdsbroker.storage.StoragePoolDomainHelper;
import org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException;
import org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class IrsProxyData {

    private static final Logger log = LoggerFactory.getLogger(IrsProxyData.class);

    // TODO the syncObj initial purpose was to lock the IrsBroker creation
    // but eventually because the IRS is single threaded and suppose to have
    // quite a load of requests,
    // In order to avoid unexpected behavior we have used the syncObj to
    // lock each request to the IRS
    // and by that we caused a serialization of requests to the IRS.
    // This lock should be removed as soon as the IrsBroker is turned
    // multi threaded
    public Object syncObj = new Object();

    private final String storagePoolRefreshJobId;
    private final String domainRecoverOnHostJobId;
    private final HashSet<Guid> triedVdssList = new HashSet<Guid>();
    private Guid currentVdsId;

    private static Set<VDSStatus> vdsConnectedToPoolStatuses;

    static {
        vdsConnectedToPoolStatuses = EnumSet.copyOf(StoragePoolDomainHelper.vdsDomainsActiveMonitoringStatus);
        vdsConnectedToPoolStatuses.addAll(StoragePoolDomainHelper.vdsDomainsMaintenanceMonitoringStatus);
        vdsConnectedToPoolStatuses.add(VDSStatus.NonResponsive);
        vdsConnectedToPoolStatuses.add(VDSStatus.PreparingForMaintenance);
    }

    private Guid preferredHostId;

    public Guid getCurrentVdsId() {
        return getIrsProxy() != null ? currentVdsId : Guid.Empty;
    }

    public void setCurrentVdsId(Guid value) {
        currentVdsId = (Guid.Empty.equals(value)) ? null : value;
    }

    private String privatemCurrentIrsHost;
    private IIrsServer irsProxy;
    private Guid fencedIrs;

    private IIrsServer getmIrsProxy() {
        return irsProxy;
    }

    private int privatemIrsPort;

    private int getmIrsPort() {
        return privatemIrsPort;
    }

    private void setmIrsPort(int value) {
        privatemIrsPort = value;
    }

    private VdsProtocol privateProtocol;

    private VdsProtocol getProtocol() {
        return this.privateProtocol;
    }

    private void setProtocol(VdsProtocol value) {
        this.privateProtocol = value;
    }

    private Version privateVersion;

    private Version getVersion() {
        return this.privateVersion;
    }

    private void setVersion(Version version) {
        this.privateVersion = version;
    }

    public Guid getFencedIrs() {
        return fencedIrs;
    }

    public void setFencedIrs(Guid fencedIrs) {
        this.fencedIrs = fencedIrs;
    }

    private Guid _storagePoolId = Guid.Empty;

    public IrsProxyData(Guid storagePoolId) {
        _storagePoolId = storagePoolId;
        int storagePoolRefreshTime = Config.<Integer> getValue(ConfigValues.StoragePoolRefreshTimeInSeconds);
        storagePoolRefreshJobId = getSchedulUtil().scheduleAFixedDelayJob(this,
                "_updatingTimer_Elapsed", new Class[0], new Object[0], storagePoolRefreshTime,
                storagePoolRefreshTime, TimeUnit.SECONDS);
        domainRecoverOnHostJobId =
                getSchedulUtil().scheduleAFixedDelayJob(this,
                        "hostsStorageConnectionsAndPoolMetadataRefresh",
                        new Class[0],
                        new Object[0],
                        Config.<Integer>getValue(ConfigValues.HostStorageConnectionAndPoolRefreshTimeInSeconds),
                        storagePoolRefreshTime,
                        TimeUnit.SECONDS);
    }

    protected SchedulerUtil getSchedulUtil() {
        return Injector.get(SchedulerUtilQuartzImpl.class);
    }

    private void updateStoragePoolStatus(Guid poolId, StoragePoolStatus status, AuditLogType auditLogType, EngineError error) {
        ResourceManager
                .getInstance()
                .getEventListener()
                .storagePoolStatusChange(poolId, status,
                        auditLogType,
                        error);
    }

    @OnTimerMethodAnnotation("_updatingTimer_Elapsed")
    public void _updatingTimer_Elapsed() {
        try {
            synchronized (syncObj) {
                if (!_disposed) {
                    StoragePool storagePool = DbFacade.getInstance().getStoragePoolDao()
                            .get(_storagePoolId);

                    if (storagePool != null) {
                        boolean poolStatusDeterminedByHostsStatus = FeatureSupported.dataCenterWithoutSpm(storagePool.getCompatibilityVersion());
                        // when there are no hosts in status up, it means that there shouldn't be domain monitoring
                        // so all the domains need to move to "unknown" status as otherwise their status won't change.
                        if (DbFacade.getInstance()
                                .getVdsDao()
                                .getAllForStoragePoolAndStatuses(_storagePoolId, StoragePoolDomainHelper.vdsDomainsActiveMonitoringStatus)
                                .isEmpty()) {
                            StoragePoolDomainHelper.updateApplicablePoolDomainsStatuses(_storagePoolId,
                                    StoragePoolDomainHelper.storageDomainMonitoredStatus,
                                    StorageDomainStatus.Unknown, "no reporting hosts");
                            // TODO: need to check if it's fine to skip the update when the status is already NonResponsive (as domains status maybe be not updated.
                            if (poolStatusDeterminedByHostsStatus && storagePool.getStatus() != StoragePoolStatus.NonResponsive) {
                                updateStoragePoolStatus(storagePool.getId(), StoragePoolStatus.NonResponsive,
                                        AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_NON_RESPONSIVE_NO_REPORTING_HOSTS,
                                        EngineError.ENGINE);
                            }
                        }  else if (poolStatusDeterminedByHostsStatus && storagePool.getStatus() != StoragePoolStatus.Up) {
                                updateStoragePoolStatus(storagePool.getId(), StoragePoolStatus.Up,
                                        AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_UP_REPORTING_HOSTS,
                                        null);
                        }

                        if (storagePool.getStatus() == StoragePoolStatus.Up ||
                                storagePool.getStatus() == StoragePoolStatus.NonResponsive ||
                                storagePool.getStatus() == StoragePoolStatus.Contend) {
                            if (!poolStatusDeterminedByHostsStatus) {
                                proceedStoragePoolStats(storagePool);
                            } else {
                                List<StorageDomain> storageDomains = DbFacade.getInstance().getStorageDomainDao()
                                        .getAllForStoragePool(storagePool.getId());
                                domainsInMaintenanceCheck(storageDomains, storagePool);
                            }
                        }
                    }

                }
            }
        } catch (Exception ex) {
        }
    }

    private int _errorAttempts;

    private static Collection<Guid> getVdsConnectedToPool(Guid storagePoolId) {
        // Note - this method is used as it returns only hosts from VIRT supported clusters
        // (we use the domain monitoring results only from those clusters hosts).
        // every change to it should be inspected carefully.
        return Entities.getIds(DbFacade.getInstance().getVdsDao().getAllForStoragePoolAndStatuses(storagePoolId,
                vdsConnectedToPoolStatuses));
    }

    @SuppressWarnings("unchecked")
    private void proceedStoragePoolStats(StoragePool storagePool) {
        // ugly patch because vdsm doesnt check if host is spm on spm
        // operations
        VDSReturnValue result = null;
        Guid curVdsId = currentVdsId;
        if (curVdsId != null) {
            result = ResourceManager.getInstance().runVdsCommand(VDSCommandType.SpmStatus,
                    new SpmStatusVDSCommandParameters(curVdsId, _storagePoolId));
        }

        if (result == null
                || !result.getSucceeded()
                || (result.getSucceeded() && ((SpmStatusResult) result.getReturnValue()).getSpmStatus() != SpmStatus.SPM)) {
            // update pool status to problematic until fence will happen
            if (storagePool.getStatus() != StoragePoolStatus.NonResponsive
                    && storagePool.getStatus() != StoragePoolStatus.NotOperational) {
                if (result != null && result.getVdsError() != null) {
                    updateStoragePoolStatus(_storagePoolId, StoragePoolStatus.NonResponsive,
                                    AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC_WITH_ERROR,
                                    result.getVdsError().getCode());
                } else {
                    updateStoragePoolStatus(_storagePoolId, StoragePoolStatus.NonResponsive,
                                    AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC,
                                    EngineError.ENGINE);
                }
            }

            // if spm status didn't work or not spm and NOT NETWORK
            // PROBLEM
            // then cause failover with attempts
            if (result != null && !(result.getExceptionObject() instanceof VDSNetworkException)) {
                HashMap<Guid, AsyncTaskStatus> tasksList =
                        (HashMap<Guid, AsyncTaskStatus>) ResourceManager
                                .getInstance()
                                .runVdsCommand(VDSCommandType.HSMGetAllTasksStatuses,
                                        new VdsIdVDSCommandParametersBase(curVdsId)).getReturnValue();
                boolean allTasksFinished = true;
                if (tasksList != null) {
                    for (AsyncTaskStatus taskStatus : tasksList.values()) {
                        if (AsyncTaskStatusEnum.finished != taskStatus.getStatus()) {
                            allTasksFinished = false;
                            break;
                        }
                    }
                }
                if ((tasksList == null) || allTasksFinished) {
                    nullifyInternalProxies();
                } else {
                    if (_errorAttempts < Config.<Integer> getValue(ConfigValues.SPMFailOverAttempts)) {
                        _errorAttempts++;
                        log.warn("failed getting spm status for pool '{}' ({}), attempt number: {}",
                                _storagePoolId, storagePool.getName(), _errorAttempts);
                    } else {
                        nullifyInternalProxies();
                        _errorAttempts = 0;
                    }
                }
            }
        } else if (result.getSucceeded()
                && ((SpmStatusResult) result.getReturnValue()).getSpmStatus() == SpmStatus.SPM
                && (storagePool.getStatus() == StoragePoolStatus.NonResponsive || storagePool.getStatus() == StoragePoolStatus.Contend)) {
            // if recovered from network exception set back to up
            DbFacade.getInstance().getStoragePoolDao().updateStatus(storagePool.getId(), StoragePoolStatus.Up);
            storagePool.setStatus(StoragePoolStatus.Up);
            ResourceManager.getInstance().getEventListener()
                    .storagePoolStatusChanged(storagePool.getId(), storagePool.getStatus());
        }
        List<StorageDomain> domainsInDb = DbFacade.getInstance().getStorageDomainDao()
                .getAllForStoragePool(_storagePoolId);
        GetStoragePoolInfoVDSCommandParameters tempVar = new GetStoragePoolInfoVDSCommandParameters(
                _storagePoolId);
        tempVar.setIgnoreFailoverLimit(true);
        VDSReturnValue storagePoolInfoResult = ResourceManager.getInstance().runVdsCommand(
                VDSCommandType.GetStoragePoolInfo, tempVar);
        if (storagePoolInfoResult.getSucceeded()) {
            KeyValuePairCompat<StoragePool, List<StorageDomain>> data =
                    (KeyValuePairCompat<StoragePool, List<StorageDomain>>) storagePoolInfoResult
                            .getReturnValue();
            int masterVersion = data.getKey().getMasterDomainVersion();
            HashSet<Guid> domainsInVds = new HashSet<Guid>();
            for (StorageDomain domainData : data.getValue()) {
                domainsInVds.add(domainData.getId());
                proceedStorageDomain(domainData, masterVersion, storagePool);
            }
            for (final StorageDomain domainInDb : domainsInDb) {
                if (domainInDb.getStorageDomainType() != StorageDomainType.Master
                        && domainInDb.getStatus() != StorageDomainStatus.Locked
                        && !domainInDb.getStorageType().isCinderDomain()
                        && !domainsInVds.contains(domainInDb.getId())) {
                    // domain not attached to pool anymore
                    DbFacade.getInstance()
                            .getStoragePoolIsoMapDao()
                            .remove(new StoragePoolIsoMapId(domainInDb.getId(),
                                    _storagePoolId));
                }

            }
        }

        domainsInMaintenanceCheck(domainsInDb, storagePool);
    }

    private void domainsInMaintenanceCheck(List<StorageDomain> storageDomains, StoragePool pool) {
        for (StorageDomain domainInDb : storageDomains) {
            if (domainInDb.getStatus() == StorageDomainStatus.PreparingForMaintenance) {
                queueDomainMaintenanceCheck(domainInDb, pool);
            }
        }
    }

    public void queueDomainMaintenanceCheck(final StorageDomain domain, final StoragePool pool) {
        getEventQueue()
                .submitEventAsync(new Event(_storagePoolId, domain.getId(), null, EventType.DOMAINFAILOVER, ""),
                        new Callable<EventResult>() {
                            @Override
                            public EventResult call() {
                                Collection<Guid> vdsConnectedToPool = getVdsConnectedToPool(_storagePoolId);
                                Set<Guid> vdsDomInMaintenance = _domainsInMaintenance.get(domain.getId());
                                if (vdsConnectedToPool.isEmpty() ||
                                        (vdsDomInMaintenance != null &&
                                                vdsDomInMaintenance.containsAll(vdsConnectedToPool))) {
                                    log.info("Moving domain '{}' to maintenance", domain.getId());
                                    DbFacade.getInstance().getStoragePoolIsoMapDao().updateStatus(
                                            domain.getStoragePoolIsoMapData().getId(),
                                            StorageDomainStatus.Maintenance);
                                    AuditLogableBase auditLogableBase = new AuditLogableBase();
                                    auditLogableBase.addCustomValue("StorageDomainName", domain.getName());
                                    auditLogableBase.addCustomValue("StoragePoolName", pool.getName());
                                    new AuditLogDirector().log(auditLogableBase, AuditLogType.STORAGE_DOMAIN_MOVED_TO_MAINTENANCE);
                                }
                                return null;
                            }
                        });
    }

    private EventQueue getEventQueue() {
        return Injector.get(EventQueue.class);
    }

    public Guid getPreferredHostId() {
        return preferredHostId;
    }

    public void setPreferredHostId(Guid preferredHostId) {
        this.preferredHostId = preferredHostId;
    }

    private void proceedStorageDomain(StorageDomain data, int dataMasterVersion, StoragePool storagePool) {
        StorageDomain storage_domain = DbFacade.getInstance().getStorageDomainDao().getForStoragePool(data.getId(), _storagePoolId);
        StorageDomainStatic domainFromDb = null;
        StoragePoolIsoMap domainPoolMap = null;

        if (storage_domain != null) {
            domainFromDb = storage_domain.getStorageStaticData();
            domainPoolMap = storage_domain.getStoragePoolIsoMapData();
            // If the domain is master in the DB
            if (domainFromDb.getStorageDomainType() == StorageDomainType.Master && domainPoolMap != null
                    && domainPoolMap.getStatus() != StorageDomainStatus.Locked) {
                // and the domain is not master in the VDSM
                if (!((data.getStorageDomainType() == StorageDomainType.Master) || (data.getStorageDomainType() == StorageDomainType.Unknown))) {
                    reconstructMasterDomainNotInSync(data.getStoragePoolId(),
                            domainFromDb.getId(),
                            "Mismatch between master in DB and VDSM",
                            MessageFormat.format("Master domain is not in sync between DB and VDSM. "
                                    + "Domain {0} marked as master in DB and not in the storage",
                                    domainFromDb.getStorageName()));
                } // or master in DB and VDSM but there is a version
                  // mismatch
                else if (dataMasterVersion != storagePool.getMasterDomainVersion()) {
                    reconstructMasterDomainNotInSync(data.getStoragePoolId(),
                            domainFromDb.getId(),
                            "Mismatch between master version in DB and VDSM",
                            MessageFormat.format("Master domain version is not in sync between DB and VDSM. "
                                    + "Domain {0} marked as master, but the version in DB: {1} and in VDSM: {2}",
                                    domainFromDb.getStorageName(),
                                    storagePool.getMasterDomainVersion(),
                                    dataMasterVersion));
                }
            }
            boolean statusChanged = false;
            if (domainPoolMap == null) {
                data.setStoragePoolId(_storagePoolId);
                DbFacade.getInstance().getStoragePoolIsoMapDao().save(data.getStoragePoolIsoMapData());
                statusChanged = true;
            } else if (domainPoolMap.getStatus() != StorageDomainStatus.Locked
                    && domainPoolMap.getStatus() != data.getStatus()) {
                if (domainPoolMap.getStatus() != StorageDomainStatus.Inactive
                        && data.getStatus() != StorageDomainStatus.Inactive) {
                    DbFacade.getInstance().getStoragePoolIsoMapDao().update(data.getStoragePoolIsoMapData());
                    statusChanged = true;
                }
                if (data.getStatus() != null && data.getStatus() == StorageDomainStatus.Inactive
                        && domainFromDb.getStorageDomainType() == StorageDomainType.Master) {
                    StoragePool pool = DbFacade.getInstance().getStoragePoolDao()
                            .get(domainPoolMap.getstorage_pool_id());
                    if (pool != null) {
                        DbFacade.getInstance().getStoragePoolDao().updateStatus(pool.getId(),
                                StoragePoolStatus.Maintenance);
                        pool.setStatus(StoragePoolStatus.Maintenance);
                        ResourceManager.getInstance().getEventListener()
                                .storagePoolStatusChanged(pool.getId(), StoragePoolStatus.Maintenance);
                    }
                }
            }

            // For block domains, synchronize LUN details comprising the storage domain with the DB
            if (statusChanged && data.getStatus() == StorageDomainStatus.Active && storage_domain.getStorageType().isBlockDomain()) {
                ResourceManager.getInstance().getEventListener().syncLunsInfoForBlockStorageDomain(<commit_msg>core: reducing race - avoid syncing the storage domain status when its in process<commit_after>package org.ovirt.engine.core.vdsbroker.irsbroker;

import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang.StringUtils;
import org.ovirt.engine.core.common.AuditLogType;
import org.ovirt.engine.core.common.FeatureSupported;
import org.ovirt.engine.core.common.businessentities.AsyncTaskStatus;
import org.ovirt.engine.core.common.businessentities.AsyncTaskStatusEnum;
import org.ovirt.engine.core.common.businessentities.BusinessEntitiesDefinitions;
import org.ovirt.engine.core.common.businessentities.Entities;
import org.ovirt.engine.core.common.businessentities.NonOperationalReason;
import org.ovirt.engine.core.common.businessentities.SpmStatus;
import org.ovirt.engine.core.common.businessentities.SpmStatusResult;
import org.ovirt.engine.core.common.businessentities.StorageDomain;
import org.ovirt.engine.core.common.businessentities.StorageDomainStatic;
import org.ovirt.engine.core.common.businessentities.StorageDomainStatus;
import org.ovirt.engine.core.common.businessentities.StorageDomainType;
import org.ovirt.engine.core.common.businessentities.StoragePool;
import org.ovirt.engine.core.common.businessentities.StoragePoolIsoMap;
import org.ovirt.engine.core.common.businessentities.StoragePoolIsoMapId;
import org.ovirt.engine.core.common.businessentities.StoragePoolStatus;
import org.ovirt.engine.core.common.businessentities.VDS;
import org.ovirt.engine.core.common.businessentities.VDSDomainsData;
import org.ovirt.engine.core.common.businessentities.VDSStatus;
import org.ovirt.engine.core.common.businessentities.VdsProtocol;
import org.ovirt.engine.core.common.businessentities.vds_spm_id_map;
import org.ovirt.engine.core.common.config.Config;
import org.ovirt.engine.core.common.config.ConfigValues;
import org.ovirt.engine.core.common.errors.EngineError;
import org.ovirt.engine.core.common.errors.EngineMessage;
import org.ovirt.engine.core.common.eventqueue.Event;
import org.ovirt.engine.core.common.eventqueue.EventQueue;
import org.ovirt.engine.core.common.eventqueue.EventResult;
import org.ovirt.engine.core.common.eventqueue.EventType;
import org.ovirt.engine.core.common.locks.LockingGroup;
import org.ovirt.engine.core.common.utils.Pair;
import org.ovirt.engine.core.common.vdscommands.ConnectStoragePoolVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.DisconnectStoragePoolVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.GetStoragePoolInfoVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStartVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStatusVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStopVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.VDSCommandType;
import org.ovirt.engine.core.common.vdscommands.VDSReturnValue;
import org.ovirt.engine.core.common.vdscommands.VdsIdVDSCommandParametersBase;
import org.ovirt.engine.core.compat.Guid;
import org.ovirt.engine.core.compat.KeyValuePairCompat;
import org.ovirt.engine.core.compat.RefObject;
import org.ovirt.engine.core.compat.TransactionScopeOption;
import org.ovirt.engine.core.compat.Version;
import org.ovirt.engine.core.dal.dbbroker.DbFacade;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase;
import org.ovirt.engine.core.di.Injector;
import org.ovirt.engine.core.utils.lock.EngineLock;
import org.ovirt.engine.core.utils.lock.LockManagerFactory;
import org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil;
import org.ovirt.engine.core.utils.timer.OnTimerMethodAnnotation;
import org.ovirt.engine.core.utils.timer.SchedulerUtil;
import org.ovirt.engine.core.utils.timer.SchedulerUtilQuartzImpl;
import org.ovirt.engine.core.utils.transaction.TransactionMethod;
import org.ovirt.engine.core.utils.transaction.TransactionSupport;
import org.ovirt.engine.core.vdsbroker.ResourceManager;
import org.ovirt.engine.core.vdsbroker.TransportFactory;
import org.ovirt.engine.core.vdsbroker.jsonrpc.JsonRpcIIrsServer;
import org.ovirt.engine.core.vdsbroker.storage.StoragePoolDomainHelper;
import org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException;
import org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class IrsProxyData {

    private static final Logger log = LoggerFactory.getLogger(IrsProxyData.class);

    // TODO the syncObj initial purpose was to lock the IrsBroker creation
    // but eventually because the IRS is single threaded and suppose to have
    // quite a load of requests,
    // In order to avoid unexpected behavior we have used the syncObj to
    // lock each request to the IRS
    // and by that we caused a serialization of requests to the IRS.
    // This lock should be removed as soon as the IrsBroker is turned
    // multi threaded
    public Object syncObj = new Object();

    private final String storagePoolRefreshJobId;
    private final String domainRecoverOnHostJobId;
    private final HashSet<Guid> triedVdssList = new HashSet<Guid>();
    private Guid currentVdsId;

    private static Set<VDSStatus> vdsConnectedToPoolStatuses;

    static {
        vdsConnectedToPoolStatuses = EnumSet.copyOf(StoragePoolDomainHelper.vdsDomainsActiveMonitoringStatus);
        vdsConnectedToPoolStatuses.addAll(StoragePoolDomainHelper.vdsDomainsMaintenanceMonitoringStatus);
        vdsConnectedToPoolStatuses.add(VDSStatus.NonResponsive);
        vdsConnectedToPoolStatuses.add(VDSStatus.PreparingForMaintenance);
    }

    private Guid preferredHostId;

    public Guid getCurrentVdsId() {
        return getIrsProxy() != null ? currentVdsId : Guid.Empty;
    }

    public void setCurrentVdsId(Guid value) {
        currentVdsId = (Guid.Empty.equals(value)) ? null : value;
    }

    private String privatemCurrentIrsHost;
    private IIrsServer irsProxy;
    private Guid fencedIrs;

    private IIrsServer getmIrsProxy() {
        return irsProxy;
    }

    private int privatemIrsPort;

    private int getmIrsPort() {
        return privatemIrsPort;
    }

    private void setmIrsPort(int value) {
        privatemIrsPort = value;
    }

    private VdsProtocol privateProtocol;

    private VdsProtocol getProtocol() {
        return this.privateProtocol;
    }

    private void setProtocol(VdsProtocol value) {
        this.privateProtocol = value;
    }

    private Version privateVersion;

    private Version getVersion() {
        return this.privateVersion;
    }

    private void setVersion(Version version) {
        this.privateVersion = version;
    }

    public Guid getFencedIrs() {
        return fencedIrs;
    }

    public void setFencedIrs(Guid fencedIrs) {
        this.fencedIrs = fencedIrs;
    }

    private Guid _storagePoolId = Guid.Empty;

    public IrsProxyData(Guid storagePoolId) {
        _storagePoolId = storagePoolId;
        int storagePoolRefreshTime = Config.<Integer> getValue(ConfigValues.StoragePoolRefreshTimeInSeconds);
        storagePoolRefreshJobId = getSchedulUtil().scheduleAFixedDelayJob(this,
                "_updatingTimer_Elapsed", new Class[0], new Object[0], storagePoolRefreshTime,
                storagePoolRefreshTime, TimeUnit.SECONDS);
        domainRecoverOnHostJobId =
                getSchedulUtil().scheduleAFixedDelayJob(this,
                        "hostsStorageConnectionsAndPoolMetadataRefresh",
                        new Class[0],
                        new Object[0],
                        Config.<Integer>getValue(ConfigValues.HostStorageConnectionAndPoolRefreshTimeInSeconds),
                        storagePoolRefreshTime,
                        TimeUnit.SECONDS);
    }

    protected SchedulerUtil getSchedulUtil() {
        return Injector.get(SchedulerUtilQuartzImpl.class);
    }

    private void updateStoragePoolStatus(Guid poolId, StoragePoolStatus status, AuditLogType auditLogType, EngineError error) {
        ResourceManager
                .getInstance()
                .getEventListener()
                .storagePoolStatusChange(poolId, status,
                        auditLogType,
                        error);
    }

    @OnTimerMethodAnnotation("_updatingTimer_Elapsed")
    public void _updatingTimer_Elapsed() {
        try {
            synchronized (syncObj) {
                if (!_disposed) {
                    StoragePool storagePool = DbFacade.getInstance().getStoragePoolDao()
                            .get(_storagePoolId);

                    if (storagePool != null) {
                        boolean poolStatusDeterminedByHostsStatus = FeatureSupported.dataCenterWithoutSpm(storagePool.getCompatibilityVersion());
                        // when there are no hosts in status up, it means that there shouldn't be domain monitoring
                        // so all the domains need to move to "unknown" status as otherwise their status won't change.
                        if (DbFacade.getInstance()
                                .getVdsDao()
                                .getAllForStoragePoolAndStatuses(_storagePoolId, StoragePoolDomainHelper.vdsDomainsActiveMonitoringStatus)
                                .isEmpty()) {
                            StoragePoolDomainHelper.updateApplicablePoolDomainsStatuses(_storagePoolId,
                                    StoragePoolDomainHelper.storageDomainMonitoredStatus,
                                    StorageDomainStatus.Unknown, "no reporting hosts");
                            // TODO: need to check if it's fine to skip the update when the status is already NonResponsive (as domains status maybe be not updated.
                            if (poolStatusDeterminedByHostsStatus && storagePool.getStatus() != StoragePoolStatus.NonResponsive) {
                                updateStoragePoolStatus(storagePool.getId(), StoragePoolStatus.NonResponsive,
                                        AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_NON_RESPONSIVE_NO_REPORTING_HOSTS,
                                        EngineError.ENGINE);
                            }
                        }  else if (poolStatusDeterminedByHostsStatus && storagePool.getStatus() != StoragePoolStatus.Up) {
                                updateStoragePoolStatus(storagePool.getId(), StoragePoolStatus.Up,
                                        AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_UP_REPORTING_HOSTS,
                                        null);
                        }

                        if (storagePool.getStatus() == StoragePoolStatus.Up ||
                                storagePool.getStatus() == StoragePoolStatus.NonResponsive ||
                                storagePool.getStatus() == StoragePoolStatus.Contend) {
                            if (!poolStatusDeterminedByHostsStatus) {
                                proceedStoragePoolStats(storagePool);
                            } else {
                                List<StorageDomain> storageDomains = DbFacade.getInstance().getStorageDomainDao()
                                        .getAllForStoragePool(storagePool.getId());
                                domainsInMaintenanceCheck(storageDomains, storagePool);
                            }
                        }
                    }

                }
            }
        } catch (Exception ex) {
        }
    }

    private int _errorAttempts;

    private static Collection<Guid> getVdsConnectedToPool(Guid storagePoolId) {
        // Note - this method is used as it returns only hosts from VIRT supported clusters
        // (we use the domain monitoring results only from those clusters hosts).
        // every change to it should be inspected carefully.
        return Entities.getIds(DbFacade.getInstance().getVdsDao().getAllForStoragePoolAndStatuses(storagePoolId,
                vdsConnectedToPoolStatuses));
    }

    @SuppressWarnings("unchecked")
    private void proceedStoragePoolStats(StoragePool storagePool) {
        // ugly patch because vdsm doesnt check if host is spm on spm
        // operations
        VDSReturnValue result = null;
        Guid curVdsId = currentVdsId;
        if (curVdsId != null) {
            result = ResourceManager.getInstance().runVdsCommand(VDSCommandType.SpmStatus,
                    new SpmStatusVDSCommandParameters(curVdsId, _storagePoolId));
        }

        if (result == null
                || !result.getSucceeded()
                || (result.getSucceeded() && ((SpmStatusResult) result.getReturnValue()).getSpmStatus() != SpmStatus.SPM)) {
            // update pool status to problematic until fence will happen
            if (storagePool.getStatus() != StoragePoolStatus.NonResponsive
                    && storagePool.getStatus() != StoragePoolStatus.NotOperational) {
                if (result != null && result.getVdsError() != null) {
                    updateStoragePoolStatus(_storagePoolId, StoragePoolStatus.NonResponsive,
                                    AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC_WITH_ERROR,
                                    result.getVdsError().getCode());
                } else {
                    updateStoragePoolStatus(_storagePoolId, StoragePoolStatus.NonResponsive,
                                    AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC,
                                    EngineError.ENGINE);
                }
            }

            // if spm status didn't work or not spm and NOT NETWORK
            // PROBLEM
            // then cause failover with attempts
            if (result != null && !(result.getExceptionObject() instanceof VDSNetworkException)) {
                HashMap<Guid, AsyncTaskStatus> tasksList =
                        (HashMap<Guid, AsyncTaskStatus>) ResourceManager
                                .getInstance()
                                .runVdsCommand(VDSCommandType.HSMGetAllTasksStatuses,
                                        new VdsIdVDSCommandParametersBase(curVdsId)).getReturnValue();
                boolean allTasksFinished = true;
                if (tasksList != null) {
                    for (AsyncTaskStatus taskStatus : tasksList.values()) {
                        if (AsyncTaskStatusEnum.finished != taskStatus.getStatus()) {
                            allTasksFinished = false;
                            break;
                        }
                    }
                }
                if ((tasksList == null) || allTasksFinished) {
                    nullifyInternalProxies();
                } else {
                    if (_errorAttempts < Config.<Integer> getValue(ConfigValues.SPMFailOverAttempts)) {
                        _errorAttempts++;
                        log.warn("failed getting spm status for pool '{}' ({}), attempt number: {}",
                                _storagePoolId, storagePool.getName(), _errorAttempts);
                    } else {
                        nullifyInternalProxies();
                        _errorAttempts = 0;
                    }
                }
            }
        } else if (result.getSucceeded()
                && ((SpmStatusResult) result.getReturnValue()).getSpmStatus() == SpmStatus.SPM
                && (storagePool.getStatus() == StoragePoolStatus.NonResponsive || storagePool.getStatus() == StoragePoolStatus.Contend)) {
            // if recovered from network exception set back to up
            DbFacade.getInstance().getStoragePoolDao().updateStatus(storagePool.getId(), StoragePoolStatus.Up);
            storagePool.setStatus(StoragePoolStatus.Up);
            ResourceManager.getInstance().getEventListener()
                    .storagePoolStatusChanged(storagePool.getId(), storagePool.getStatus());
        }
        List<StorageDomain> domainsInDb = DbFacade.getInstance().getStorageDomainDao()
                .getAllForStoragePool(_storagePoolId);
        GetStoragePoolInfoVDSCommandParameters tempVar = new GetStoragePoolInfoVDSCommandParameters(
                _storagePoolId);
        tempVar.setIgnoreFailoverLimit(true);
        VDSReturnValue storagePoolInfoResult = ResourceManager.getInstance().runVdsCommand(
                VDSCommandType.GetStoragePoolInfo, tempVar);
        if (storagePoolInfoResult.getSucceeded()) {
            KeyValuePairCompat<StoragePool, List<StorageDomain>> data =
                    (KeyValuePairCompat<StoragePool, List<StorageDomain>>) storagePoolInfoResult
                            .getReturnValue();
            int masterVersion = data.getKey().getMasterDomainVersion();
            HashSet<Guid> domainsInVds = new HashSet<Guid>();
            for (StorageDomain domainData : data.getValue()) {
                domainsInVds.add(domainData.getId());
                proceedStorageDomain(domainData, masterVersion, storagePool);
            }
            for (final StorageDomain domainInDb : domainsInDb) {
                if (domainInDb.getStorageDomainType() != StorageDomainType.Master
                        && domainInDb.getStatus() != StorageDomainStatus.Locked
                        && !domainInDb.getStorageType().isCinderDomain()
                        && !domainsInVds.contains(domainInDb.getId())) {
                    // domain not attached to pool anymore
                    DbFacade.getInstance()
                            .getStoragePoolIsoMapDao()
                            .remove(new StoragePoolIsoMapId(domainInDb.getId(),
                                    _storagePoolId));
                }

            }
        }

        domainsInMaintenanceCheck(domainsInDb, storagePool);
    }

    private void domainsInMaintenanceCheck(List<StorageDomain> storageDomains, StoragePool pool) {
        for (StorageDomain domainInDb : storageDomains) {
            if (domainInDb.getStatus() == StorageDomainStatus.PreparingForMaintenance) {
                queueDomainMaintenanceCheck(domainInDb, pool);
            }
        }
    }

    public void queueDomainMaintenanceCheck(final StorageDomain domain, final StoragePool pool) {
        getEventQueue()
                .submitEventAsync(new Event(_storagePoolId, domain.getId(), null, EventType.DOMAINFAILOVER, ""),
                        new Callable<EventResult>() {
                            @Override
                            public EventResult call() {
                                Collection<Guid> vdsConnectedToPool = getVdsConnectedToPool(_storagePoolId);
                                Set<Guid> vdsDomInMaintenance = _domainsInMaintenance.get(domain.getId());
                                if (vdsConnectedToPool.isEmpty() ||
                                        (vdsDomInMaintenance != null &&
                                                vdsDomInMaintenance.containsAll(vdsConnectedToPool))) {
                                    log.info("Moving domain '{}' to maintenance", domain.getId());
                                    DbFacade.getInstance().getStoragePoolIsoMapDao().updateStatus(
                                            domain.getStoragePoolIsoMapData().getId(),
                                            StorageDomainStatus.Maintenance);
                                    AuditLogableBase auditLogableBase = new AuditLogableBase();
                                    auditLogableBase.addCustomValue("StorageDomainName", domain.getName());
                                    auditLogableBase.addCustomValue("StoragePoolName", pool.getName());
                                    new AuditLogDirector().log(auditLogableBase, AuditLogType.STORAGE_DOMAIN_MOVED_TO_MAINTENANCE);
                                }
                                return null;
                            }
                        });
    }

    private EventQueue getEventQueue() {
        return Injector.get(EventQueue.class);
    }

    public Guid getPreferredHostId() {
        return preferredHostId;
    }

    public void setPreferredHostId(Guid preferredHostId) {
        this.preferredHostId = preferredHostId;
    }

    private void proceedStorageDomain(StorageDomain data, int dataMasterVersion, StoragePool storagePool) {
        StorageDomain storage_domain = DbFacade.getInstance().getStorageDomainDao().getForStoragePool(data.getId(), _storagePoolId);
        StorageDomainStatic domainFromDb = null;
        StoragePoolIsoMap domainPoolMap = null;

        if (storage_domain != null) {
            domainFromDb = storage_domain.getStorageStaticData();
            domainPoolMap = storage_domain.getStoragePoolIsoMapData();
            // If the domain is master in the DB
            if (domainFromDb.getStorageDomainType() == StorageDomainType.Master && domainPoolMap != null
                    && domainPoolMap.getStatus() != StorageDomainStatus.Locked) {
                // and the domain is not master in the VDSM
                if (!((data.getStorageDomainType() == StorageDomainType.Master) || (data.getStorageDomainType() == StorageDomainType.Unknown))) {
                    reconstructMasterDomainNotInSync(data.getStoragePoolId(),
                            domainFromDb.getId(),
                            "Mismatch between master in DB and VDSM",
                            MessageFormat.format("Master domain is not in sync between DB and VDSM. "
                                    + "Domain {0} marked as master in DB and not in the storage",
                                    domainFromDb.getStorageName()));
                } // or master in DB and VDSM but there is a version
                  // mismatch
                else if (dataMasterVersion != storagePool.getMasterDomainVersion()) {
                    reconstructMasterDomainNotInSync(data.getStoragePoolId(),
                            domainFromDb.getId(),
                            "Mismatch between master version in DB and VDSM",
                            MessageFormat.format("Master domain version is not in sync between DB and VDSM. "
                                    + "Domain {0} marked as master, but the version in DB: {1} and in VDSM: {2}",
                                    domainFromDb.getStorageName(),
                                    storagePool.getMasterDomainVersion(),
                                    dataMasterVersion));
                }
            }
            boolean statusChanged = false;
            if (domainPoolMap == null) {
                data.setStoragePoolId(_storagePoolId);
                DbFacade.getInstance().getStoragePoolIsoMapDao().save(data.getStoragePoolIsoMapData());
                statusChanged = true;
            } else if (!domainPoolMap.getStatus().isStorageDomainInProcess()
                    && domainPoolMap.getStatus() != data.getStatus()) {
                if (domainPoolMap.getStatus() != StorageDomainStatus.Inactive
                        && data.getStatus() != StorageDomainStatus.Inactive) {
                    DbFacade.getInstance().getStoragePoolIsoMapDao().update(data.getStoragePoolIsoMapData());
                    statusChanged = true;
                }
                if (data.getStatus() != null && data.getStatus() == StorageDomainStatus.Inactive
                        && domainFromDb.getStorageDomainType() == StorageDomainType.Master) {
                    StoragePool pool = DbFacade.getInstance().getStoragePoolDao()
                            .get(domainPoolMap.getstorage_pool_id());
                    if (pool != null) {
                        DbFacade.getInstance().getStoragePoolDao().updateStatus(pool.getId(),
                                StoragePoolStatus.Maintenance);
                        pool.setStatus(StoragePoolStatus.Maintenance);
                        ResourceManager.getInstance().getEventListener()
                                .storagePoolStatusChanged(pool.getId(), StoragePoolStatus.Maintenance);
                    }
                }
            }

            // For block domains, synchronize LUN details comprising the storage domain with the DB
            if (statusChanged && data.getStatus() == StorageDomainStatus.Active && storage_domain.getStorageType().isBlockDomain()) {
                ResourceManager.getInstance().getEventListener().syncLunsInfoForBlockStorageDomain(
{'fraction_non_alphanumeric': 0.08163635630468971, 'fraction_numerical': 0.000443566272833582, 'mean_word_length': 2.194782608695652, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 68, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3234453', 'n_tokens_mistral': 12642, 'n_tokens_neox': 12275, 'n_words': 2228}