<commit_before>package org.ovirt.engine.core.vdsbroker.irsbroker;

import java.lang.reflect.UndeclaredThrowableException;
import java.net.SocketException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.ovirt.engine.core.common.AuditLogType;
import org.ovirt.engine.core.common.businessentities.AsyncTaskStatus;
import org.ovirt.engine.core.common.businessentities.AsyncTaskStatusEnum;
import org.ovirt.engine.core.common.businessentities.BusinessEntitiesDefinitions;
import org.ovirt.engine.core.common.businessentities.NonOperationalReason;
import org.ovirt.engine.core.common.businessentities.SpmStatus;
import org.ovirt.engine.core.common.businessentities.SpmStatusResult;
import org.ovirt.engine.core.common.businessentities.StorageDomainStatus;
import org.ovirt.engine.core.common.businessentities.StorageDomainType;
import org.ovirt.engine.core.common.businessentities.StoragePoolIsoMapId;
import org.ovirt.engine.core.common.businessentities.StoragePoolStatus;
import org.ovirt.engine.core.common.businessentities.VDS;
import org.ovirt.engine.core.common.businessentities.VDSDomainsData;
import org.ovirt.engine.core.common.businessentities.VDSStatus;
import org.ovirt.engine.core.common.businessentities.StorageDomainStatic;
import org.ovirt.engine.core.common.businessentities.StorageDomain;
import org.ovirt.engine.core.common.businessentities.storage_pool;
import org.ovirt.engine.core.common.businessentities.StoragePoolIsoMap;
import org.ovirt.engine.core.common.businessentities.vds_spm_id_map;
import org.ovirt.engine.core.common.config.Config;
import org.ovirt.engine.core.common.config.ConfigValues;
import org.ovirt.engine.core.common.errors.VDSError;
import org.ovirt.engine.core.common.errors.VdcBllErrors;
import org.ovirt.engine.core.common.eventqueue.Event;
import org.ovirt.engine.core.common.eventqueue.EventQueue;
import org.ovirt.engine.core.common.eventqueue.EventResult;
import org.ovirt.engine.core.common.eventqueue.EventType;
import org.ovirt.engine.core.common.utils.Pair;
import org.ovirt.engine.core.common.vdscommands.ConnectStoragePoolVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.DisconnectStoragePoolVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.GetStoragePoolInfoVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.IrsBaseVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStartVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStatusVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStopVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.VDSCommandType;
import org.ovirt.engine.core.common.vdscommands.VDSReturnValue;
import org.ovirt.engine.core.common.vdscommands.VdsIdVDSCommandParametersBase;
import org.ovirt.engine.core.compat.Guid;
import org.ovirt.engine.core.compat.KeyValuePairCompat;
import org.ovirt.engine.core.compat.RefObject;
import org.ovirt.engine.core.compat.TransactionScopeOption;
import org.ovirt.engine.core.dal.dbbroker.DbFacade;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase;
import org.ovirt.engine.core.utils.ejb.BeanProxyType;
import org.ovirt.engine.core.utils.ejb.BeanType;
import org.ovirt.engine.core.utils.ejb.EjbUtils;
import org.ovirt.engine.core.utils.log.Log;
import org.ovirt.engine.core.utils.log.LogFactory;
import org.ovirt.engine.core.utils.log.Logged;
import org.ovirt.engine.core.utils.log.Logged.LogLevel;
import org.ovirt.engine.core.utils.log.LoggedUtils;
import org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil;
import org.ovirt.engine.core.utils.timer.OnTimerMethodAnnotation;
import org.ovirt.engine.core.utils.timer.SchedulerUtilQuartzImpl;
import org.ovirt.engine.core.utils.transaction.TransactionMethod;
import org.ovirt.engine.core.utils.transaction.TransactionSupport;
import org.ovirt.engine.core.vdsbroker.ResourceManager;
import org.ovirt.engine.core.vdsbroker.vdsbroker.BrokerCommandBase;
import org.ovirt.engine.core.vdsbroker.vdsbroker.VDSExceptionBase;
import org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException;
import org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcRunTimeException;
import org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcStruct;
import org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils;

@Logged(errorLevel = LogLevel.ERROR)
public abstract class IrsBrokerCommand<P extends IrsBaseVDSCommandParameters> extends BrokerCommandBase<P> {
    public static final long BYTES_TO_GB = 1024 * 1024 * 1024;
    private static Map<Guid, IrsProxyData> _irsProxyData = new ConcurrentHashMap<Guid, IrsProxyData>();

    public static void UpdateVdsDomainsData(Guid vdsId, String vdsName, Guid storagePoolId,
            java.util.ArrayList<VDSDomainsData> vdsDomainData) {
        IrsProxyData proxy = _irsProxyData.get(storagePoolId);
        if (proxy != null) {
            proxy.UpdateVdsDomainsData(vdsId, vdsName, vdsDomainData);
        }
    }

    public static boolean isDomainsReportedAsProblematic(Guid storagePoolId, List<VDSDomainsData> vdsDomainsData) {
        IrsProxyData proxy = _irsProxyData.get(storagePoolId);
        if (proxy != null) {
            return proxy.isDomainsReportedAsProblematic(vdsDomainsData);
        }
        return false;
    }

    @Override
    protected VDSExceptionBase createDefaultConcreteException(String errorMessage) {
        return new IRSErrorException(errorMessage);
    }

    public static void Init() {
        for (storage_pool sp : DbFacade.getInstance().getStoragePoolDao().getAll()) {
            if (!_irsProxyData.containsKey(sp.getId())) {
                _irsProxyData.put(sp.getId(), new IrsProxyData(sp.getId()));
            }
        }
    }

    public void RemoveIrsProxy() {
        _irsProxyData.get(getParameters().getStoragePoolId()).Dispose();
        _irsProxyData.remove(getParameters().getStoragePoolId());
    }

    protected static class IrsProxyData {
        // TODO the syncObj initial purposs was to lock the IrsBroker creation
        // but eventually because the IRS is singlethreaded and suppose to have
        // quite a load of requests,
        // In order to avoid unexpected behaviour we have used the syncObj to
        // lock each request to the IRS
        // and by that we caused a searialization of requests to the IRS.
        // This lock should be removed as soon as the IrsBroker is turned
        // multithreaded
        public Object syncObj = new Object();

        private final String storagePoolRefreshJobId;
        private final java.util.HashSet<Guid> mTriedVdssList = new java.util.HashSet<Guid>();
        private Guid mCurrentVdsId;

        public Guid getCurrentVdsId() {
            return getIrsProxy() != null ? mCurrentVdsId : Guid.Empty;
        }

        public void setCurrentVdsId(Guid value) {
            mCurrentVdsId = (Guid.Empty.equals(value)) ? null : value;
        }

        private String privatemCurrentIrsHost;
        private IIrsServer privatemIrsProxy;

        private IIrsServer getmIrsProxy() {
            return privatemIrsProxy;
        }

        private int privatemIrsPort;

        private int getmIrsPort() {
            return privatemIrsPort;
        }

        private void setmIrsPort(int value) {
            privatemIrsPort = value;
        }

        private Guid _storagePoolId = Guid.Empty;
        private String mIsoPrefix = "";

        public IrsProxyData(Guid storagePoolId) {
            _storagePoolId = storagePoolId;
            int storagePoolRefreshTime = Config.<Integer> GetValue(ConfigValues.StoragePoolRefreshTimeInSeconds);
            storagePoolRefreshJobId = SchedulerUtilQuartzImpl.getInstance().scheduleAFixedDelayJob(this,
                    "_updatingTimer_Elapsed", new Class[0], new Object[0], storagePoolRefreshTime,
                    storagePoolRefreshTime, TimeUnit.SECONDS);
        }

        @OnTimerMethodAnnotation("_updatingTimer_Elapsed")
        public void _updatingTimer_Elapsed() {
            try {
                synchronized (syncObj) {
                    if (!_disposed) {
                        storage_pool storagePool = DbFacade.getInstance().getStoragePoolDao()
                                .get(_storagePoolId);
                        if (storagePool != null
                                && (storagePool.getstatus() == StoragePoolStatus.Up
                                        || storagePool.getstatus() == StoragePoolStatus.Problematic || storagePool
                                        .getstatus() == StoragePoolStatus.Contend)) {
                            proceedStoragePoolStats(storagePool);
                        }

                    }
                }
            } catch (java.lang.Exception ex) {
            }
        }

        private int _errorAttempts;

        @SuppressWarnings("unchecked")
        private void proceedStoragePoolStats(storage_pool storagePool) {
            // ugly patch because vdsm doesnt check if host is spm on spm
            // operations
            VDSReturnValue result = null;
            if (mCurrentVdsId != null) {
                result = ResourceManager.getInstance().runVdsCommand(VDSCommandType.SpmStatus,
                        new SpmStatusVDSCommandParameters(mCurrentVdsId, _storagePoolId));
            }

            if (result == null
                    || !result.getSucceeded()
                    || (result.getSucceeded() && ((SpmStatusResult) result.getReturnValue()).getSpmStatus() != SpmStatus.SPM)) {
                // update pool status to problematic until fence will happen
                if (storagePool.getstatus() != StoragePoolStatus.Problematic
                        && storagePool.getstatus() != StoragePoolStatus.NotOperational) {
                    if (result != null && result.getVdsError() != null) {
                        ResourceManager
                                .getInstance()
                                .getEventListener()
                                .storagePoolStatusChange(_storagePoolId, StoragePoolStatus.Problematic,
                                        AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC_WITH_ERROR,
                                        result.getVdsError().getCode());
                    } else {
                        ResourceManager
                                .getInstance()
                                .getEventListener()
                                .storagePoolStatusChange(_storagePoolId, StoragePoolStatus.Problematic,
                                        AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC,
                                        VdcBllErrors.ENGINE);
                    }
                }

                // if spm status didnt work or not spm and NOT NETWORK
                // PROBLEM
                // then cause failover with attempts
                if (result != null && !(result.getExceptionObject() instanceof VDSNetworkException)) {
                    HashMap<Guid, AsyncTaskStatus> tasksList =
                            (HashMap<Guid, AsyncTaskStatus>) ResourceManager
                                    .getInstance()
                                    .runVdsCommand(VDSCommandType.HSMGetAllTasksStatuses,
                                            new VdsIdVDSCommandParametersBase(mCurrentVdsId)).getReturnValue();
                    boolean allTasksFinished = true;
                    if (tasksList != null) {
                        for (AsyncTaskStatus taskStatus : tasksList.values()) {
                            if (AsyncTaskStatusEnum.finished != taskStatus.getStatus()) {
                                allTasksFinished = false;
                                break;
                            }
                        }
                    }
                    if ((tasksList == null) || allTasksFinished) {
                        nullifyInternalProxies();
                    } else {
                        if (_errorAttempts < Config.<Integer> GetValue(ConfigValues.SPMFailOverAttempts)) {
                            _errorAttempts++;
                            log.warnFormat("failed getting spm status for pool {0}:{1}, attempt number {2}",
                                    _storagePoolId, storagePool.getname(), _errorAttempts);
                        } else {
                            nullifyInternalProxies();
                            _errorAttempts = 0;
                        }
                    }
                }
            } else if (result != null
                    && result.getSucceeded()
                    && ((SpmStatusResult) result.getReturnValue()).getSpmStatus() == SpmStatus.SPM
                    && (storagePool.getstatus() == StoragePoolStatus.Problematic || storagePool.getstatus() == StoragePoolStatus.Contend)) {
                // if recovered from network exception set back to up
                DbFacade.getInstance().getStoragePoolDao().updateStatus(storagePool.getId(), StoragePoolStatus.Up);
                storagePool.setstatus(StoragePoolStatus.Up);
                ResourceManager.getInstance().getEventListener()
                        .storagePoolStatusChanged(storagePool.getId(), storagePool.getstatus());
            }
            GetStoragePoolInfoVDSCommandParameters tempVar = new GetStoragePoolInfoVDSCommandParameters(
                    _storagePoolId);
            tempVar.setIgnoreFailoverLimit(true);
            VDSReturnValue storagePoolInfoResult = ResourceManager.getInstance().runVdsCommand(
                    VDSCommandType.GetStoragePoolInfo, tempVar);
            if (storagePoolInfoResult.getSucceeded()) {
                KeyValuePairCompat<storage_pool, java.util.List<StorageDomain>> data =
                        (KeyValuePairCompat<storage_pool, java.util.List<StorageDomain>>) storagePoolInfoResult
                                .getReturnValue();
                int masterVersion = data.getKey().getmaster_domain_version();
                java.util.HashSet<Guid> domainsInVds = new java.util.HashSet<Guid>();
                for (StorageDomain domainData : data.getValue()) {
                    domainsInVds.add(domainData.getId());
                    proceedStorageDomain(domainData, masterVersion, storagePool);
                }
                List<StorageDomain> domainsInDb = DbFacade.getInstance().getStorageDomainDao()
                        .getAllForStoragePool(_storagePoolId);

                for (final StorageDomain domainInDb : domainsInDb) {
                    if (domainInDb.getStorageDomainType() != StorageDomainType.Master
                            && domainInDb.getStatus() != StorageDomainStatus.Locked
                            && !domainsInVds.contains(domainInDb.getId())) {
                        // domain not attached to pool anymore
                        DbFacade.getInstance()
                                .getStoragePoolIsoMapDao()
                                .remove(new StoragePoolIsoMapId(domainInDb.getId(),
                                        _storagePoolId));
                    }
                }
            }
        }

        private void proceedStorageDomain(StorageDomain data, int dataMasterVersion, storage_pool storagePool) {
            StorageDomain storage_domain = DbFacade.getInstance().getStorageDomainDao().getForStoragePool(data.getId(), _storagePoolId);
            StorageDomainStatic domainFromDb = null;
            StoragePoolIsoMap domainPoolMap = null;

            if (storage_domain != null) {
                domainFromDb = storage_domain.getStorageStaticData();
                domainPoolMap = storage_domain.getStoragePoolIsoMapData();
                // If the domain is master in the DB
                if (domainFromDb.getStorageDomainType() == StorageDomainType.Master && domainPoolMap != null
                        && domainPoolMap.getstatus() != StorageDomainStatus.Locked) {
                    // and the domain is not master in the VDSM
                    if (!((data.getStorageDomainType() == StorageDomainType.Master) || (data.getStorageDomainType() == StorageDomainType.Unknown))) {
                        reconstructMasterDomainNotInSync(data.getStoragePoolId().getValue(),
                                domainFromDb.getId(),
                                "Mismatch between master in DB and VDSM",
                                MessageFormat.format("Master domain is not in sync between DB and VDSM. "
                                        + "Domain {0} marked as master in DB and not in the storage",
                                        domainFromDb.getStorageName()));
                    } // or master in DB and VDSM but there is a version
                      // mismatch
                    else if (dataMasterVersion != storagePool.getmaster_domain_version()) {
                        reconstructMasterDomainNotInSync(data.getStoragePoolId().getValue(),
                                domainFromDb.getId(),
                                "Mismatch between master version in DB and VDSM",
                                MessageFormat.format("Master domain version is not in sync between DB and VDSM. "
                                        + "Domain {0} marked as master, but the version in DB: {1} and in VDSM: {2}",
                                        domainFromDb.getStorageName(),
                                        storagePool.getmaster_domain_version(),
                                        dataMasterVersion));
                    }
                }
                boolean statusChanged = false;
                if (domainPoolMap == null) {
                    data.setStoragePoolId(_storagePoolId);
                    DbFacade.getInstance().getStoragePoolIsoMapDao().save(data.getStoragePoolIsoMapData());
                    statusChanged = true;
                } else if (domainPoolMap.getstatus() != StorageDomainStatus.Locked
                        && domainPoolMap.getstatus() != data.getStatus()) {
                    if (domainPoolMap.getstatus() != StorageDomainStatus.InActive
                            && data.getStatus() != StorageDomainStatus.InActive) {
                        DbFacade.getInstance().getStoragePoolIsoMapDao().update(data.getStoragePoolIsoMapData());
                        statusChanged = true;
                    }
                    if (data.getStatus() != null && data.getStatus() == StorageDomainStatus.InActive
                            && domainFromDb.getStorageDomainType() == StorageDomainType.Master) {
                        storage_pool pool = DbFacade.getInstance().getStoragePoolDao()
                                .get(domainPoolMap.getstorage_pool_id().getValue());
                        if (pool != null) {
                            DbFacade.getInstance().getStoragePoolDao().updateStatus(pool.getId(),StoragePoolStatus.Maintenance);
                            pool.setstatus(StoragePoolStatus.Maintenance);
                            ResourceManager.getInstance().getEventListener()
                                    .storagePoolStatusChanged(pool.getId(), StoragePoolStatus.Maintenance);
                        }
                    }
                }
                // if status didn't change and still not active no need to
                // update dynamic data
                if (statusChanged
                        || (domainPoolMap.getstatus() != StorageDomainStatus.InActive && data.getStatus() == StorageDomainStatus.Active)) {
                    DbFacade.getInstance().getStorageDomainDynamicDao().update(data.getStorageDynamicData());
                    if (data.getAvailableDiskSize() != null && data.getUsedDiskSize() != null) {
                        double freePercent = data.getStorageDynamicData().getfreeDiskPercent();
                        int freeDiskInGB = data.getStorageDynamicData().getfreeDiskInGB();
                        AuditLogType type = AuditLogType.UNASSIGNED;
                        boolean spaceThresholdMet =
                                freeDiskInGB <= Config.<Integer> GetValue(ConfigValues.FreeSpaceCriticalLowInGB);
                        boolean percentThresholdMet =
                                freePercent <= Config.<Integer> GetValue(ConfigValues.FreeSpaceLow);
                        if (spaceThresholdMet && percentThresholdMet) {
                            type = AuditLogType.IRS_DISK_SPACE_LOW_ERROR;
                        } else {
                            if (spaceThresholdMet || percentThresholdMet) {
                                type = AuditLogType.IRS_DISK_SPACE_LOW;
                            }
                        }
                        if (type != AuditLogType.UNASSIGNED) {
                            AuditLogableBase logable = new AuditLogableBase();
                            logable.setStorageDomain(data);
                            logable.setStoragePoolId(_storagePoolId);
                            logable.addCustomValue("DiskSpace", (data.getAvailableDiskSize()).toString());
                            data.setStorageName(domainFromDb.getStorageName());
                            AuditLogDirector.log(logable, type);

                        }
                    }

                    Set<VdcBllErrors> alerts = data.getAlerts();
                    if (alerts != null && !alerts.isEmpty()) {

                        AuditLogableBase logable = new AuditLogableBase();
                        logable.setStorageDomain(data);
                        data.setStorageName(domainFromDb.getStorageName());
                        logable.setStoragePoolId(_storagePoolId);

                        for (VdcBllErrors alert : alerts) {
                            switch (alert) {
                            case VG_METADATA_CRITICALLY_FULL:
                                AuditLogDirector.log(logable, AuditLogType.STORAGE_ALERT_VG_METADATA_CRITICALLY_FULL);
                                break;
                            case SMALL_VG_METADATA:
                                AuditLogDirector.log(logable, AuditLogType.STORAGE_ALERT_SMALL_VG_METADATA);
                                break;
                            default:
                                log.errorFormat("Unrecognized alert for domain {0}(id = {1}): {2}",
                                        data.getStorageName(),
                                        data.getId(),
                                        alert);
                                break;
                            }
                        }
                    }
                }

            } else {
                log.debugFormat("The domain with id {0} was not found in DB", data.getId());
            }
        }

        /**
         * Reconstructs the master domain when the old domain is not in sync.
         *
         * @param storagePoolId
         *            The storage pool id.
         * @param masterDomainId
         *            The master domain id.
         * @param exceptionMessage
         *            The message of the exception to throw.
         * @param logMessage
         *            The log message to write in the log.
         */
        private void reconstructMasterDomainNotInSync(final Guid storagePoolId,
                final Guid masterDomainId,
                final String exceptionMessage,
                final String logMessage) {

            ((EventQueue) EjbUtils.findBean(BeanType.EVENTQUEUE_MANAGER, BeanProxyType.LOCAL)).submitEventSync(new Event(_storagePoolId,
                    masterDomainId, null, EventType.RECONSTRUCT),
                    new Callable<EventResult>() {
                        @Override
                        public EventResult call() {
                            log.warnFormat(logMessage);

                            AuditLogableBase logable = new AuditLogableBase(mCurrentVdsId);
                            logable.setStorageDomainId(masterDomainId);
                            AuditLogDirector.log(logable, AuditLogType.SYSTEM_MASTER_DOMAIN_NOT_IN_SYNC);

                            return ResourceManager.getInstance()
                                    .getEventListener()
                                    .masterDomainNotOperational(masterDomainId, storagePoolId);

                        }
                    });
            throw new IRSNoMasterDomainException(exceptionMessage);
        }

        public java.util.HashSet<Guid> getTriedVdssList() {
            return mTriedVdssList;
        }

        public void Init(VDS vds) {
            mCurrentVdsId = vds.getId();
            setmIrsPort(vds.getPort());
            privatemCurrentIrsHost = vds.getHostName();
        }

        public boolean failover() {
            Guid vdsId = mCurrentVdsId;
            nullifyInternalProxies();
            boolean performFailover = false;
            if (vdsId != null) {
                try {
                    VDSReturnValue statusResult = ResourceManager.getInstance().runVdsCommand(VDSCommandType.SpmStatus,
                            new SpmStatusVDSCommandParameters(vdsId, _storagePoolId));
                    if (statusResult != null
                            && statusResult.getSucceeded()
                            && (((SpmStatusResult) statusResult.getReturnValue()).getSpmStatus() == SpmStatus.SPM || ((SpmStatusResult) statusResult
                                    .getReturnValue()).getSpmStatus() == SpmStatus.Contend)) {
                        performFailover = ResourceManager
                                .getInstance()
                                .runVdsCommand(VDSCommandType.SpmStop,
                                        new SpmStopVDSCommandParameters(vdsId, _storagePoolId)).getSucceeded();
                    } else {
                        performFailover = true;
                    }
                } catch (Exception ex) {
                    // try to failover to another host if failed to get spm
                    // status or stop spm
                    // (in case mCurrentVdsId has wrong id for some reason)
                    log.errorFormat("Could not get spm status on host {0} for spmStop.", vdsId);
                    performFailover = true;
                }
            }

            if (performFailover) {
                log.infoFormat("Irs placed on server {0} failed. Proceed Failover", vdsId);
                mTriedVdssList.add(vdsId);
                return true;
            } else {
                log.errorFormat("IRS failover failed - cant allocate vds server");
                return false;
            }
        }

        public IIrsServer getIrsProxy() {
            if (getmIrsProxy() == null) {
                final storage_pool storagePool = DbFacade.getInstance().getStoragePoolDao().get(_storagePoolId);
                // don't try to start spm on uninitialized pool
                if (storagePool.getstatus() != StoragePoolStatus.Uninitialized) {
                    String host =
                            TransactionSupport.executeInScope(TransactionScopeOption.Suppress,
                                    new TransactionMethod<String>() {
                                        @Override
                                        public String runInTransaction() {
                                            return gethostFromVds();
                                        }
                                    });

                    if (host != null) {
                        // Get the values of the timeouts:
                        int clientTimeOut = Config.<Integer> GetValue(ConfigValues.vdsTimeout) * 1000;
                        int connectionTimeOut = Config.<Integer>GetValue(ConfigValues.vdsConnectionTimeout) * 1000;
                        int clientRetries = Config.<Integer> GetValue(ConfigValues.vdsRetries);

                        Pair<IrsServerConnector, HttpClient> returnValue =
                                XmlRpcUtils.getConnection(host,
                                        getmIrsPort(),
                                        clientTimeOut,
                                        connectionTimeOut,
                                        clientRetries,
                                        IrsServerConnector.class,
                                        Config.<Boolean> GetValue(ConfigValues.UseSecureConnectionWithServers));
                        privatemIrsProxy = new IrsServerWrapper(returnValue.getFirst(), returnValue.getSecond());
                        runStoragePoolUpEvent(storagePool);
                    }
                }
            }
            return getmIrsProxy();
        }

        private void runStoragePoolUpEvent(final storage_pool storagePool) {
            ThreadPoolUtil.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        if (DbFacade.getInstance().isStoragePoolMasterUp(_storagePoolId)) {
                            ResourceManager.getInstance()
                                    .getEventListener()
                                    .storagePoolUpEvent(storagePool, _isSpmStartCalled);
                        }
                    } catch (RuntimeException exp) {
                        log.error("Error in StoragePoolUpEvent - ", exp);
                    }

                }
            });
        }

        /**
         * Returns True if there are other vdss in pool
         */
        public boolean getHasVdssForSpmSelection() {
            return (getPrioritizedVdsInPool().size() > 0);
        }

        private String gethostFromVds() {
            _isSpmStartCalled = false;
            String returnValue = null;
            Guid curVdsId = (mCurrentVdsId != null) ? mCurrentVdsId : Guid.Empty;
            storage_pool storagePool = DbFacade.getInstance().getStoragePoolDao().get(_storagePoolId);

            if (storagePool == null) {
                log.infoFormat("hostFromVds::Finished elect spm, storage pool {0} was removed", _storagePoolId);
                return null;
            }

            List<VDS> prioritizedVdsInPool = getPrioritizedVdsInPool();
            mCurrentVdsId = null;

            // If VDS is in initialize status, wait for it to be up (or until
            // configurable timeout is reached)
            waitForVdsIfIsInitializing(curVdsId);
            // update pool status to problematic while selecting spm
            StoragePoolStatus prevStatus = storagePool.getstatus();
            if (prevStatus != StoragePoolStatus.Problematic) {
                try {
                    ResourceManager
                            .getInstance()
                            .getEventListener()
                            .storagePoolStatusChange(_storagePoolId, StoragePoolStatus.Problematic,
                                    AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC_SEARCHING_NEW_SPM,
                                    VdcBllErrors.ENGINE, TransactionScopeOption.RequiresNew);
                } catch (RuntimeException ex) {
                    throw new IRSStoragePoolStatusException(ex);
                }
            }
            VDS selectedVds = null;
            SpmStatusResult spmStatus = null;

            if (prioritizedVdsInPool != null && prioritizedVdsInPool.size() > 0) {
                selectedVds = prioritizedVdsInPool.get(0);
            } else if (!Guid.Empty.equals(curVdsId) && !getTriedVdssList().contains(curVdsId)) {
                selectedVds = DbFacade.getInstance().getVdsDao().get(curVdsId);
                if (selectedVds.getStatus() != VDSStatus.Up
                        || selectedVds.getVdsSpmPriority() == BusinessEntitiesDefinitions.HOST_MIN_SPM_PRIORITY) {
                    selectedVds = null;
                }
            }

            if (selectedVds != null) {
                // Stores origin host id in case and will be needed to disconnect from storage pool
                Guid selectedVdsId = selectedVds.getId();
                Integer selectedVdsSpmId = selectedVds.getVdsSpmId();
                mTriedVdssList.add(selectedVdsId);

                VDSReturnValue returnValueFromVds = ResourceManager.getInstance().runVdsCommand(
                        VDSCommandType.SpmStatus,
                        new SpmStatusVDSCommandParameters(selectedVds.getId(), _storagePoolId));
                spmStatus = (SpmStatusResult) returnValueFromVds.getReturnValue();
                if (spmStatus != null) {
                    mCurrentVdsId = selectedVds.getId();
                    boolean performedPoolConnect = false;
                    log.infoFormat("hostFromVds::selectedVds - {0}, spmStatus {1}, storage pool {2}",
                            selectedVds.getName(), spmStatus.getSpmStatus().toString(), storagePool.getname());
                    if (spmStatus.getSpmStatus() == SpmStatus.Unknown_Pool) {
                        Guid masterId = DbFacade.getInstance().getStorageDomainDao()
                                .getMasterStorageDomainIdForPool(_storagePoolId);
                        VDSReturnValue connectResult = ResourceManager.getInstance().runVdsCommand(
                                VDSCommandType.ConnectStoragePool,
                                new ConnectStoragePoolVDSCommandParameters(selectedVds.getId(), _storagePoolId,
                                        selectedVds.getVdsSpmId(), masterId, storagePool.getmaster_domain_version()));
                        if (!connectResult.getSucceeded()
                                && connectResult.getExceptionObject() instanceof IRSNoMasterDomainException) {
                            throw connectResult.getExceptionObject();
                        } else if (!connectResult.getSucceeded()) {
                            // if connect to pool fails throw exception for
                            // failover
                            throw new IRSNonOperationalException("Could not connect host to Data Center(Storage issue)");
                        }
                        performedPoolConnect = true;
                        // refresh spmStatus result
                        spmStatus = (SpmStatusResult) ResourceManager
                                .getInstance()
                                .runVdsCommand(VDSCommandType.SpmStatus,
                                        new SpmStatusVDSCommandParameters(selectedVds.getId(), _storagePoolId))
                                .getReturnValue();
                        log.infoFormat(
                                "hostFromVds::Connected host to pool - selectedVds - {0}, spmStatus {1}, storage pool {2}",
                                selectedVds.getName(),
                                spmStatus.getSpmStatus().toString(),
                                storagePool.getname());
                    }
                    RefObject<VDS> tempRefObject = new RefObject<VDS>(selectedVds);
                    spmStatus =
                            handleSpmStatusResult(curVdsId, prioritizedVdsInPool, storagePool, tempRefObject, spmStatus);
                    selectedVds = tempRefObject.argvalue;

                    if (selectedVds != null) {
                        RefObject<VDS> tempRefObject2 = new RefObject<VDS>(selectedVds);
                        RefObject<SpmStatusResult> tempRefObject3 = new RefObject<SpmStatusResult>(spmStatus);
                        returnValue = handleSelectedVdsForSPM(storagePool, tempRefObject2, tempRefObject3, prevStatus);
                        selectedVds = tempRefObject2.argvalue;
                        spmStatus = tempRefObject3.argvalue;
                    } else {
                        mCurrentVdsId = null;
                    }
                    if (performedPoolConnect && selectedVds == null) {
                        // if could not start spm on this host and connected to
                        // pool here
                        // then disconnect
                        ResourceManager.getInstance().runVdsCommand(
                                VDSCommandType.DisconnectStoragePool,
                                new DisconnectStoragePoolVDSCommandParameters(selectedVdsId, _storagePoolId,
                                        selectedVdsSpmId));
                    }
                } else {

                    log.infoFormat("hostFromVds::selectedVds - {0}, spmStatus returned null!",
                            selectedVds.getName());
                    if (returnValueFromVds.getExceptionObject() instanceof IRSNoMasterDomainException) {
                        throw returnValueFromVds.getExceptionObject();
                    }
                }
            }
            return returnValue;
        }

        private List<VDS> getPrioritizedVdsInPool() {
            Guid curVdsId = (mCurrentVdsId != null) ? mCurrentVdsId : Guid.Empty;
            // Gets a list of the hosts in the storagePool, that are "UP", ordered
            // by vds_spm_priority (not including -1) and secondly ordered by RANDOM(), to
            // deal with the case that there are several hosts with the same priority.
            List<VDS> allVds = DbFacade.getInstance().getVdsDao().getListForSpmSelection(_storagePoolId);
            List<VDS> vdsRelevantForSpmSelection = new ArrayList<VDS>();
            for (VDS vds : allVds) {
                if (!mTriedVdssList.contains(vds.getId()) && !vds.getId().equals(curVdsId)) {
                    vdsRelevantForSpmSelection.add(vds);
                }
            }
            return vdsRelevantForSpmSelection;
        }

        private boolean _isSpmStartCalled;

        private String handleSelectedVdsForSPM(storage_pool storagePool, RefObject<VDS> selectedVds,
                                               RefObject<SpmStatusResult> spmStatus, StoragePoolStatus prevStatus) {
            String returnValue = null;
            if (spmStatus.argvalue == null || spmStatus.argvalue.getSpmStatus() != SpmStatus.SPM) {
                movePoolToProblematicInDB(storagePool, true);

                selectedVds.argvalue = null;
                log.infoFormat(
                        "spm start treatment ended and status is not SPM!!! status: {0} - setting selectedVds to null!",
                        spmStatus.argvalue.getSpmStatus().toString());
            } else {
                Init(selectedVds.argvalue);
                storagePool.setLVER(spmStatus.argvalue.getSpmLVER());
                storagePool.setspm_vds_id(selectedVds.argvalue.getId());
                // if were problemtaic or not operational and succeeded to find
                // host move pool to up
                if (prevStatus != StoragePoolStatus.NotOperational && prevStatus != StoragePoolStatus.Problematic) {
                    storagePool.setstatus(prevStatus);
                } else {
                    storagePool.setstatus(StoragePoolStatus.Up);
                }
                DbFacade.getInstance().getStoragePoolDao().update(storagePool);
                ResourceManager.getInstance()
                        .getEventListener()
                        .storagePoolStatusChanged(storagePool.getId(), storagePool.getstatus());

                returnValue = selectedVds.argvalue.getHostName();
                log.infoFormat("Initialize Irs proxy from vds: {0}", returnValue);
                AuditLogableBase logable = new AuditLogableBase(selectedVds.argvalue.getId());
                logable.addCustomValue("ServerIp", returnValue);
                AuditLogDirector.log(logable, AuditLogType.IRS_HOSTED_ON_VDS);
            }
            return returnValue;
        }

        /**
         * Waits for VDS if is initializing.
         *
         * @param curVdsId
         */
        private void waitForVdsIfIsInitializing(Guid curVdsId) {
            if (!Guid.Empty.equals(curVdsId)
                    && DbFacade.getInstance().getVdsDao().get(curVdsId).getStatus() == VDSStatus.Initializing) {
                final int DELAY = 5;// 5 Sec
                int total = 0;
                Integer maxSecToWait = Config.GetValue(ConfigValues.WaitForVdsInitInSec);
                String vdsName = DbFacade.getInstance().getVdsDao().get(curVdsId).getName();
                while (total <= maxSecToWait
                        && DbFacade.getInstance().getVdsDao().get(curVdsId).getStatus() == VDSStatus.Initializing) {
                    try {
                        Thread.sleep(DELAY * 1000);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                        // exit the while block
                        break;
                    }
                    total += DELAY;
                    log.infoFormat("Waiting to Host {0} to finish initialization for {1} Sec.", vdsName, total);
                }
            }
        }

        private void movePoolToProblematicInDB(storage_pool storagePool, boolean resetSpmInDB) {
            ResourceManager
                    .getInstance()
                    .getEventListener()
                    .storagePoolStatusChange(storagePool.getId(), StoragePoolStatus.Problematic,
                            AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC, VdcBllErrors.ENGINE);

            if (resetSpmInDB) {
                storagePool.setspm_vds_id(null);
                DbFacade.getInstance().getStoragePoolDao().update(storagePool);
            }
        }

        private SpmStatusResult handleSpmStatusResult(Guid curVdsId,
                                                      List<VDS> vdsByPool,
                                                      final storage_pool storagePool,
                                                      RefObject<VDS> selectedVds,
                                                      SpmStatusResult spmStatus) {<commit_msg>engine: Vds initializyng- query improvements<commit_after>package org.ovirt.engine.core.vdsbroker.irsbroker;

import java.lang.reflect.UndeclaredThrowableException;
import java.net.SocketException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.ovirt.engine.core.common.AuditLogType;
import org.ovirt.engine.core.common.businessentities.AsyncTaskStatus;
import org.ovirt.engine.core.common.businessentities.AsyncTaskStatusEnum;
import org.ovirt.engine.core.common.businessentities.BusinessEntitiesDefinitions;
import org.ovirt.engine.core.common.businessentities.NonOperationalReason;
import org.ovirt.engine.core.common.businessentities.SpmStatus;
import org.ovirt.engine.core.common.businessentities.SpmStatusResult;
import org.ovirt.engine.core.common.businessentities.StorageDomainStatus;
import org.ovirt.engine.core.common.businessentities.StorageDomainType;
import org.ovirt.engine.core.common.businessentities.StoragePoolIsoMapId;
import org.ovirt.engine.core.common.businessentities.StoragePoolStatus;
import org.ovirt.engine.core.common.businessentities.VDS;
import org.ovirt.engine.core.common.businessentities.VDSDomainsData;
import org.ovirt.engine.core.common.businessentities.VDSStatus;
import org.ovirt.engine.core.common.businessentities.StorageDomainStatic;
import org.ovirt.engine.core.common.businessentities.StorageDomain;
import org.ovirt.engine.core.common.businessentities.storage_pool;
import org.ovirt.engine.core.common.businessentities.StoragePoolIsoMap;
import org.ovirt.engine.core.common.businessentities.vds_spm_id_map;
import org.ovirt.engine.core.common.config.Config;
import org.ovirt.engine.core.common.config.ConfigValues;
import org.ovirt.engine.core.common.errors.VDSError;
import org.ovirt.engine.core.common.errors.VdcBllErrors;
import org.ovirt.engine.core.common.eventqueue.Event;
import org.ovirt.engine.core.common.eventqueue.EventQueue;
import org.ovirt.engine.core.common.eventqueue.EventResult;
import org.ovirt.engine.core.common.eventqueue.EventType;
import org.ovirt.engine.core.common.utils.Pair;
import org.ovirt.engine.core.common.vdscommands.ConnectStoragePoolVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.DisconnectStoragePoolVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.GetStoragePoolInfoVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.IrsBaseVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStartVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStatusVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.SpmStopVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.VDSCommandType;
import org.ovirt.engine.core.common.vdscommands.VDSReturnValue;
import org.ovirt.engine.core.common.vdscommands.VdsIdVDSCommandParametersBase;
import org.ovirt.engine.core.compat.Guid;
import org.ovirt.engine.core.compat.KeyValuePairCompat;
import org.ovirt.engine.core.compat.RefObject;
import org.ovirt.engine.core.compat.TransactionScopeOption;
import org.ovirt.engine.core.dal.dbbroker.DbFacade;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase;
import org.ovirt.engine.core.utils.ejb.BeanProxyType;
import org.ovirt.engine.core.utils.ejb.BeanType;
import org.ovirt.engine.core.utils.ejb.EjbUtils;
import org.ovirt.engine.core.utils.log.Log;
import org.ovirt.engine.core.utils.log.LogFactory;
import org.ovirt.engine.core.utils.log.Logged;
import org.ovirt.engine.core.utils.log.Logged.LogLevel;
import org.ovirt.engine.core.utils.log.LoggedUtils;
import org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil;
import org.ovirt.engine.core.utils.timer.OnTimerMethodAnnotation;
import org.ovirt.engine.core.utils.timer.SchedulerUtilQuartzImpl;
import org.ovirt.engine.core.utils.transaction.TransactionMethod;
import org.ovirt.engine.core.utils.transaction.TransactionSupport;
import org.ovirt.engine.core.vdsbroker.ResourceManager;
import org.ovirt.engine.core.vdsbroker.vdsbroker.BrokerCommandBase;
import org.ovirt.engine.core.vdsbroker.vdsbroker.VDSExceptionBase;
import org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException;
import org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcRunTimeException;
import org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcStruct;
import org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils;

@Logged(errorLevel = LogLevel.ERROR)
public abstract class IrsBrokerCommand<P extends IrsBaseVDSCommandParameters> extends BrokerCommandBase<P> {
    public static final long BYTES_TO_GB = 1024 * 1024 * 1024;
    private static Map<Guid, IrsProxyData> _irsProxyData = new ConcurrentHashMap<Guid, IrsProxyData>();

    public static void UpdateVdsDomainsData(Guid vdsId, String vdsName, Guid storagePoolId,
            java.util.ArrayList<VDSDomainsData> vdsDomainData) {
        IrsProxyData proxy = _irsProxyData.get(storagePoolId);
        if (proxy != null) {
            proxy.UpdateVdsDomainsData(vdsId, vdsName, vdsDomainData);
        }
    }

    public static boolean isDomainsReportedAsProblematic(Guid storagePoolId, List<VDSDomainsData> vdsDomainsData) {
        IrsProxyData proxy = _irsProxyData.get(storagePoolId);
        if (proxy != null) {
            return proxy.isDomainsReportedAsProblematic(vdsDomainsData);
        }
        return false;
    }

    @Override
    protected VDSExceptionBase createDefaultConcreteException(String errorMessage) {
        return new IRSErrorException(errorMessage);
    }

    public static void Init() {
        for (storage_pool sp : DbFacade.getInstance().getStoragePoolDao().getAll()) {
            if (!_irsProxyData.containsKey(sp.getId())) {
                _irsProxyData.put(sp.getId(), new IrsProxyData(sp.getId()));
            }
        }
    }

    public void RemoveIrsProxy() {
        _irsProxyData.get(getParameters().getStoragePoolId()).Dispose();
        _irsProxyData.remove(getParameters().getStoragePoolId());
    }

    protected static class IrsProxyData {
        // TODO the syncObj initial purposs was to lock the IrsBroker creation
        // but eventually because the IRS is singlethreaded and suppose to have
        // quite a load of requests,
        // In order to avoid unexpected behaviour we have used the syncObj to
        // lock each request to the IRS
        // and by that we caused a searialization of requests to the IRS.
        // This lock should be removed as soon as the IrsBroker is turned
        // multithreaded
        public Object syncObj = new Object();

        private final String storagePoolRefreshJobId;
        private final java.util.HashSet<Guid> mTriedVdssList = new java.util.HashSet<Guid>();
        private Guid mCurrentVdsId;

        public Guid getCurrentVdsId() {
            return getIrsProxy() != null ? mCurrentVdsId : Guid.Empty;
        }

        public void setCurrentVdsId(Guid value) {
            mCurrentVdsId = (Guid.Empty.equals(value)) ? null : value;
        }

        private String privatemCurrentIrsHost;
        private IIrsServer privatemIrsProxy;

        private IIrsServer getmIrsProxy() {
            return privatemIrsProxy;
        }

        private int privatemIrsPort;

        private int getmIrsPort() {
            return privatemIrsPort;
        }

        private void setmIrsPort(int value) {
            privatemIrsPort = value;
        }

        private Guid _storagePoolId = Guid.Empty;
        private String mIsoPrefix = "";

        public IrsProxyData(Guid storagePoolId) {
            _storagePoolId = storagePoolId;
            int storagePoolRefreshTime = Config.<Integer> GetValue(ConfigValues.StoragePoolRefreshTimeInSeconds);
            storagePoolRefreshJobId = SchedulerUtilQuartzImpl.getInstance().scheduleAFixedDelayJob(this,
                    "_updatingTimer_Elapsed", new Class[0], new Object[0], storagePoolRefreshTime,
                    storagePoolRefreshTime, TimeUnit.SECONDS);
        }

        @OnTimerMethodAnnotation("_updatingTimer_Elapsed")
        public void _updatingTimer_Elapsed() {
            try {
                synchronized (syncObj) {
                    if (!_disposed) {
                        storage_pool storagePool = DbFacade.getInstance().getStoragePoolDao()
                                .get(_storagePoolId);
                        if (storagePool != null
                                && (storagePool.getstatus() == StoragePoolStatus.Up
                                        || storagePool.getstatus() == StoragePoolStatus.Problematic || storagePool
                                        .getstatus() == StoragePoolStatus.Contend)) {
                            proceedStoragePoolStats(storagePool);
                        }

                    }
                }
            } catch (java.lang.Exception ex) {
            }
        }

        private int _errorAttempts;

        @SuppressWarnings("unchecked")
        private void proceedStoragePoolStats(storage_pool storagePool) {
            // ugly patch because vdsm doesnt check if host is spm on spm
            // operations
            VDSReturnValue result = null;
            if (mCurrentVdsId != null) {
                result = ResourceManager.getInstance().runVdsCommand(VDSCommandType.SpmStatus,
                        new SpmStatusVDSCommandParameters(mCurrentVdsId, _storagePoolId));
            }

            if (result == null
                    || !result.getSucceeded()
                    || (result.getSucceeded() && ((SpmStatusResult) result.getReturnValue()).getSpmStatus() != SpmStatus.SPM)) {
                // update pool status to problematic until fence will happen
                if (storagePool.getstatus() != StoragePoolStatus.Problematic
                        && storagePool.getstatus() != StoragePoolStatus.NotOperational) {
                    if (result != null && result.getVdsError() != null) {
                        ResourceManager
                                .getInstance()
                                .getEventListener()
                                .storagePoolStatusChange(_storagePoolId, StoragePoolStatus.Problematic,
                                        AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC_WITH_ERROR,
                                        result.getVdsError().getCode());
                    } else {
                        ResourceManager
                                .getInstance()
                                .getEventListener()
                                .storagePoolStatusChange(_storagePoolId, StoragePoolStatus.Problematic,
                                        AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC,
                                        VdcBllErrors.ENGINE);
                    }
                }

                // if spm status didnt work or not spm and NOT NETWORK
                // PROBLEM
                // then cause failover with attempts
                if (result != null && !(result.getExceptionObject() instanceof VDSNetworkException)) {
                    HashMap<Guid, AsyncTaskStatus> tasksList =
                            (HashMap<Guid, AsyncTaskStatus>) ResourceManager
                                    .getInstance()
                                    .runVdsCommand(VDSCommandType.HSMGetAllTasksStatuses,
                                            new VdsIdVDSCommandParametersBase(mCurrentVdsId)).getReturnValue();
                    boolean allTasksFinished = true;
                    if (tasksList != null) {
                        for (AsyncTaskStatus taskStatus : tasksList.values()) {
                            if (AsyncTaskStatusEnum.finished != taskStatus.getStatus()) {
                                allTasksFinished = false;
                                break;
                            }
                        }
                    }
                    if ((tasksList == null) || allTasksFinished) {
                        nullifyInternalProxies();
                    } else {
                        if (_errorAttempts < Config.<Integer> GetValue(ConfigValues.SPMFailOverAttempts)) {
                            _errorAttempts++;
                            log.warnFormat("failed getting spm status for pool {0}:{1}, attempt number {2}",
                                    _storagePoolId, storagePool.getname(), _errorAttempts);
                        } else {
                            nullifyInternalProxies();
                            _errorAttempts = 0;
                        }
                    }
                }
            } else if (result != null
                    && result.getSucceeded()
                    && ((SpmStatusResult) result.getReturnValue()).getSpmStatus() == SpmStatus.SPM
                    && (storagePool.getstatus() == StoragePoolStatus.Problematic || storagePool.getstatus() == StoragePoolStatus.Contend)) {
                // if recovered from network exception set back to up
                DbFacade.getInstance().getStoragePoolDao().updateStatus(storagePool.getId(), StoragePoolStatus.Up);
                storagePool.setstatus(StoragePoolStatus.Up);
                ResourceManager.getInstance().getEventListener()
                        .storagePoolStatusChanged(storagePool.getId(), storagePool.getstatus());
            }
            GetStoragePoolInfoVDSCommandParameters tempVar = new GetStoragePoolInfoVDSCommandParameters(
                    _storagePoolId);
            tempVar.setIgnoreFailoverLimit(true);
            VDSReturnValue storagePoolInfoResult = ResourceManager.getInstance().runVdsCommand(
                    VDSCommandType.GetStoragePoolInfo, tempVar);
            if (storagePoolInfoResult.getSucceeded()) {
                KeyValuePairCompat<storage_pool, java.util.List<StorageDomain>> data =
                        (KeyValuePairCompat<storage_pool, java.util.List<StorageDomain>>) storagePoolInfoResult
                                .getReturnValue();
                int masterVersion = data.getKey().getmaster_domain_version();
                java.util.HashSet<Guid> domainsInVds = new java.util.HashSet<Guid>();
                for (StorageDomain domainData : data.getValue()) {
                    domainsInVds.add(domainData.getId());
                    proceedStorageDomain(domainData, masterVersion, storagePool);
                }
                List<StorageDomain> domainsInDb = DbFacade.getInstance().getStorageDomainDao()
                        .getAllForStoragePool(_storagePoolId);

                for (final StorageDomain domainInDb : domainsInDb) {
                    if (domainInDb.getStorageDomainType() != StorageDomainType.Master
                            && domainInDb.getStatus() != StorageDomainStatus.Locked
                            && !domainsInVds.contains(domainInDb.getId())) {
                        // domain not attached to pool anymore
                        DbFacade.getInstance()
                                .getStoragePoolIsoMapDao()
                                .remove(new StoragePoolIsoMapId(domainInDb.getId(),
                                        _storagePoolId));
                    }
                }
            }
        }

        private void proceedStorageDomain(StorageDomain data, int dataMasterVersion, storage_pool storagePool) {
            StorageDomain storage_domain = DbFacade.getInstance().getStorageDomainDao().getForStoragePool(data.getId(), _storagePoolId);
            StorageDomainStatic domainFromDb = null;
            StoragePoolIsoMap domainPoolMap = null;

            if (storage_domain != null) {
                domainFromDb = storage_domain.getStorageStaticData();
                domainPoolMap = storage_domain.getStoragePoolIsoMapData();
                // If the domain is master in the DB
                if (domainFromDb.getStorageDomainType() == StorageDomainType.Master && domainPoolMap != null
                        && domainPoolMap.getstatus() != StorageDomainStatus.Locked) {
                    // and the domain is not master in the VDSM
                    if (!((data.getStorageDomainType() == StorageDomainType.Master) || (data.getStorageDomainType() == StorageDomainType.Unknown))) {
                        reconstructMasterDomainNotInSync(data.getStoragePoolId().getValue(),
                                domainFromDb.getId(),
                                "Mismatch between master in DB and VDSM",
                                MessageFormat.format("Master domain is not in sync between DB and VDSM. "
                                        + "Domain {0} marked as master in DB and not in the storage",
                                        domainFromDb.getStorageName()));
                    } // or master in DB and VDSM but there is a version
                      // mismatch
                    else if (dataMasterVersion != storagePool.getmaster_domain_version()) {
                        reconstructMasterDomainNotInSync(data.getStoragePoolId().getValue(),
                                domainFromDb.getId(),
                                "Mismatch between master version in DB and VDSM",
                                MessageFormat.format("Master domain version is not in sync between DB and VDSM. "
                                        + "Domain {0} marked as master, but the version in DB: {1} and in VDSM: {2}",
                                        domainFromDb.getStorageName(),
                                        storagePool.getmaster_domain_version(),
                                        dataMasterVersion));
                    }
                }
                boolean statusChanged = false;
                if (domainPoolMap == null) {
                    data.setStoragePoolId(_storagePoolId);
                    DbFacade.getInstance().getStoragePoolIsoMapDao().save(data.getStoragePoolIsoMapData());
                    statusChanged = true;
                } else if (domainPoolMap.getstatus() != StorageDomainStatus.Locked
                        && domainPoolMap.getstatus() != data.getStatus()) {
                    if (domainPoolMap.getstatus() != StorageDomainStatus.InActive
                            && data.getStatus() != StorageDomainStatus.InActive) {
                        DbFacade.getInstance().getStoragePoolIsoMapDao().update(data.getStoragePoolIsoMapData());
                        statusChanged = true;
                    }
                    if (data.getStatus() != null && data.getStatus() == StorageDomainStatus.InActive
                            && domainFromDb.getStorageDomainType() == StorageDomainType.Master) {
                        storage_pool pool = DbFacade.getInstance().getStoragePoolDao()
                                .get(domainPoolMap.getstorage_pool_id().getValue());
                        if (pool != null) {
                            DbFacade.getInstance().getStoragePoolDao().updateStatus(pool.getId(),StoragePoolStatus.Maintenance);
                            pool.setstatus(StoragePoolStatus.Maintenance);
                            ResourceManager.getInstance().getEventListener()
                                    .storagePoolStatusChanged(pool.getId(), StoragePoolStatus.Maintenance);
                        }
                    }
                }
                // if status didn't change and still not active no need to
                // update dynamic data
                if (statusChanged
                        || (domainPoolMap.getstatus() != StorageDomainStatus.InActive && data.getStatus() == StorageDomainStatus.Active)) {
                    DbFacade.getInstance().getStorageDomainDynamicDao().update(data.getStorageDynamicData());
                    if (data.getAvailableDiskSize() != null && data.getUsedDiskSize() != null) {
                        double freePercent = data.getStorageDynamicData().getfreeDiskPercent();
                        int freeDiskInGB = data.getStorageDynamicData().getfreeDiskInGB();
                        AuditLogType type = AuditLogType.UNASSIGNED;
                        boolean spaceThresholdMet =
                                freeDiskInGB <= Config.<Integer> GetValue(ConfigValues.FreeSpaceCriticalLowInGB);
                        boolean percentThresholdMet =
                                freePercent <= Config.<Integer> GetValue(ConfigValues.FreeSpaceLow);
                        if (spaceThresholdMet && percentThresholdMet) {
                            type = AuditLogType.IRS_DISK_SPACE_LOW_ERROR;
                        } else {
                            if (spaceThresholdMet || percentThresholdMet) {
                                type = AuditLogType.IRS_DISK_SPACE_LOW;
                            }
                        }
                        if (type != AuditLogType.UNASSIGNED) {
                            AuditLogableBase logable = new AuditLogableBase();
                            logable.setStorageDomain(data);
                            logable.setStoragePoolId(_storagePoolId);
                            logable.addCustomValue("DiskSpace", (data.getAvailableDiskSize()).toString());
                            data.setStorageName(domainFromDb.getStorageName());
                            AuditLogDirector.log(logable, type);

                        }
                    }

                    Set<VdcBllErrors> alerts = data.getAlerts();
                    if (alerts != null && !alerts.isEmpty()) {

                        AuditLogableBase logable = new AuditLogableBase();
                        logable.setStorageDomain(data);
                        data.setStorageName(domainFromDb.getStorageName());
                        logable.setStoragePoolId(_storagePoolId);

                        for (VdcBllErrors alert : alerts) {
                            switch (alert) {
                            case VG_METADATA_CRITICALLY_FULL:
                                AuditLogDirector.log(logable, AuditLogType.STORAGE_ALERT_VG_METADATA_CRITICALLY_FULL);
                                break;
                            case SMALL_VG_METADATA:
                                AuditLogDirector.log(logable, AuditLogType.STORAGE_ALERT_SMALL_VG_METADATA);
                                break;
                            default:
                                log.errorFormat("Unrecognized alert for domain {0}(id = {1}): {2}",
                                        data.getStorageName(),
                                        data.getId(),
                                        alert);
                                break;
                            }
                        }
                    }
                }

            } else {
                log.debugFormat("The domain with id {0} was not found in DB", data.getId());
            }
        }

        /**
         * Reconstructs the master domain when the old domain is not in sync.
         *
         * @param storagePoolId
         *            The storage pool id.
         * @param masterDomainId
         *            The master domain id.
         * @param exceptionMessage
         *            The message of the exception to throw.
         * @param logMessage
         *            The log message to write in the log.
         */
        private void reconstructMasterDomainNotInSync(final Guid storagePoolId,
                final Guid masterDomainId,
                final String exceptionMessage,
                final String logMessage) {

            ((EventQueue) EjbUtils.findBean(BeanType.EVENTQUEUE_MANAGER, BeanProxyType.LOCAL)).submitEventSync(new Event(_storagePoolId,
                    masterDomainId, null, EventType.RECONSTRUCT),
                    new Callable<EventResult>() {
                        @Override
                        public EventResult call() {
                            log.warnFormat(logMessage);

                            AuditLogableBase logable = new AuditLogableBase(mCurrentVdsId);
                            logable.setStorageDomainId(masterDomainId);
                            AuditLogDirector.log(logable, AuditLogType.SYSTEM_MASTER_DOMAIN_NOT_IN_SYNC);

                            return ResourceManager.getInstance()
                                    .getEventListener()
                                    .masterDomainNotOperational(masterDomainId, storagePoolId);

                        }
                    });
            throw new IRSNoMasterDomainException(exceptionMessage);
        }

        public java.util.HashSet<Guid> getTriedVdssList() {
            return mTriedVdssList;
        }

        public void Init(VDS vds) {
            mCurrentVdsId = vds.getId();
            setmIrsPort(vds.getPort());
            privatemCurrentIrsHost = vds.getHostName();
        }

        public boolean failover() {
            Guid vdsId = mCurrentVdsId;
            nullifyInternalProxies();
            boolean performFailover = false;
            if (vdsId != null) {
                try {
                    VDSReturnValue statusResult = ResourceManager.getInstance().runVdsCommand(VDSCommandType.SpmStatus,
                            new SpmStatusVDSCommandParameters(vdsId, _storagePoolId));
                    if (statusResult != null
                            && statusResult.getSucceeded()
                            && (((SpmStatusResult) statusResult.getReturnValue()).getSpmStatus() == SpmStatus.SPM || ((SpmStatusResult) statusResult
                                    .getReturnValue()).getSpmStatus() == SpmStatus.Contend)) {
                        performFailover = ResourceManager
                                .getInstance()
                                .runVdsCommand(VDSCommandType.SpmStop,
                                        new SpmStopVDSCommandParameters(vdsId, _storagePoolId)).getSucceeded();
                    } else {
                        performFailover = true;
                    }
                } catch (Exception ex) {
                    // try to failover to another host if failed to get spm
                    // status or stop spm
                    // (in case mCurrentVdsId has wrong id for some reason)
                    log.errorFormat("Could not get spm status on host {0} for spmStop.", vdsId);
                    performFailover = true;
                }
            }

            if (performFailover) {
                log.infoFormat("Irs placed on server {0} failed. Proceed Failover", vdsId);
                mTriedVdssList.add(vdsId);
                return true;
            } else {
                log.errorFormat("IRS failover failed - cant allocate vds server");
                return false;
            }
        }

        public IIrsServer getIrsProxy() {
            if (getmIrsProxy() == null) {
                final storage_pool storagePool = DbFacade.getInstance().getStoragePoolDao().get(_storagePoolId);
                // don't try to start spm on uninitialized pool
                if (storagePool.getstatus() != StoragePoolStatus.Uninitialized) {
                    String host =
                            TransactionSupport.executeInScope(TransactionScopeOption.Suppress,
                                    new TransactionMethod<String>() {
                                        @Override
                                        public String runInTransaction() {
                                            return gethostFromVds();
                                        }
                                    });

                    if (host != null) {
                        // Get the values of the timeouts:
                        int clientTimeOut = Config.<Integer> GetValue(ConfigValues.vdsTimeout) * 1000;
                        int connectionTimeOut = Config.<Integer>GetValue(ConfigValues.vdsConnectionTimeout) * 1000;
                        int clientRetries = Config.<Integer> GetValue(ConfigValues.vdsRetries);

                        Pair<IrsServerConnector, HttpClient> returnValue =
                                XmlRpcUtils.getConnection(host,
                                        getmIrsPort(),
                                        clientTimeOut,
                                        connectionTimeOut,
                                        clientRetries,
                                        IrsServerConnector.class,
                                        Config.<Boolean> GetValue(ConfigValues.UseSecureConnectionWithServers));
                        privatemIrsProxy = new IrsServerWrapper(returnValue.getFirst(), returnValue.getSecond());
                        runStoragePoolUpEvent(storagePool);
                    }
                }
            }
            return getmIrsProxy();
        }

        private void runStoragePoolUpEvent(final storage_pool storagePool) {
            ThreadPoolUtil.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        if (DbFacade.getInstance().isStoragePoolMasterUp(_storagePoolId)) {
                            ResourceManager.getInstance()
                                    .getEventListener()
                                    .storagePoolUpEvent(storagePool, _isSpmStartCalled);
                        }
                    } catch (RuntimeException exp) {
                        log.error("Error in StoragePoolUpEvent - ", exp);
                    }

                }
            });
        }

        /**
         * Returns True if there are other vdss in pool
         */
        public boolean getHasVdssForSpmSelection() {
            return (getPrioritizedVdsInPool().size() > 0);
        }

        private String gethostFromVds() {
            _isSpmStartCalled = false;
            String returnValue = null;
            Guid curVdsId = (mCurrentVdsId != null) ? mCurrentVdsId : Guid.Empty;
            storage_pool storagePool = DbFacade.getInstance().getStoragePoolDao().get(_storagePoolId);

            if (storagePool == null) {
                log.infoFormat("hostFromVds::Finished elect spm, storage pool {0} was removed", _storagePoolId);
                return null;
            }

            List<VDS> prioritizedVdsInPool = getPrioritizedVdsInPool();
            mCurrentVdsId = null;

            // If VDS is in initialize status, wait for it to be up (or until
            // configurable timeout is reached)
            waitForVdsIfIsInitializing(curVdsId);
            // update pool status to problematic while selecting spm
            StoragePoolStatus prevStatus = storagePool.getstatus();
            if (prevStatus != StoragePoolStatus.Problematic) {
                try {
                    ResourceManager
                            .getInstance()
                            .getEventListener()
                            .storagePoolStatusChange(_storagePoolId, StoragePoolStatus.Problematic,
                                    AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC_SEARCHING_NEW_SPM,
                                    VdcBllErrors.ENGINE, TransactionScopeOption.RequiresNew);
                } catch (RuntimeException ex) {
                    throw new IRSStoragePoolStatusException(ex);
                }
            }
            VDS selectedVds = null;
            SpmStatusResult spmStatus = null;

            if (prioritizedVdsInPool != null && prioritizedVdsInPool.size() > 0) {
                selectedVds = prioritizedVdsInPool.get(0);
            } else if (!Guid.Empty.equals(curVdsId) && !getTriedVdssList().contains(curVdsId)) {
                selectedVds = DbFacade.getInstance().getVdsDao().get(curVdsId);
                if (selectedVds.getStatus() != VDSStatus.Up
                        || selectedVds.getVdsSpmPriority() == BusinessEntitiesDefinitions.HOST_MIN_SPM_PRIORITY) {
                    selectedVds = null;
                }
            }

            if (selectedVds != null) {
                // Stores origin host id in case and will be needed to disconnect from storage pool
                Guid selectedVdsId = selectedVds.getId();
                Integer selectedVdsSpmId = selectedVds.getVdsSpmId();
                mTriedVdssList.add(selectedVdsId);

                VDSReturnValue returnValueFromVds = ResourceManager.getInstance().runVdsCommand(
                        VDSCommandType.SpmStatus,
                        new SpmStatusVDSCommandParameters(selectedVds.getId(), _storagePoolId));
                spmStatus = (SpmStatusResult) returnValueFromVds.getReturnValue();
                if (spmStatus != null) {
                    mCurrentVdsId = selectedVds.getId();
                    boolean performedPoolConnect = false;
                    log.infoFormat("hostFromVds::selectedVds - {0}, spmStatus {1}, storage pool {2}",
                            selectedVds.getName(), spmStatus.getSpmStatus().toString(), storagePool.getname());
                    if (spmStatus.getSpmStatus() == SpmStatus.Unknown_Pool) {
                        Guid masterId = DbFacade.getInstance().getStorageDomainDao()
                                .getMasterStorageDomainIdForPool(_storagePoolId);
                        VDSReturnValue connectResult = ResourceManager.getInstance().runVdsCommand(
                                VDSCommandType.ConnectStoragePool,
                                new ConnectStoragePoolVDSCommandParameters(selectedVds.getId(), _storagePoolId,
                                        selectedVds.getVdsSpmId(), masterId, storagePool.getmaster_domain_version()));
                        if (!connectResult.getSucceeded()
                                && connectResult.getExceptionObject() instanceof IRSNoMasterDomainException) {
                            throw connectResult.getExceptionObject();
                        } else if (!connectResult.getSucceeded()) {
                            // if connect to pool fails throw exception for
                            // failover
                            throw new IRSNonOperationalException("Could not connect host to Data Center(Storage issue)");
                        }
                        performedPoolConnect = true;
                        // refresh spmStatus result
                        spmStatus = (SpmStatusResult) ResourceManager
                                .getInstance()
                                .runVdsCommand(VDSCommandType.SpmStatus,
                                        new SpmStatusVDSCommandParameters(selectedVds.getId(), _storagePoolId))
                                .getReturnValue();
                        log.infoFormat(
                                "hostFromVds::Connected host to pool - selectedVds - {0}, spmStatus {1}, storage pool {2}",
                                selectedVds.getName(),
                                spmStatus.getSpmStatus().toString(),
                                storagePool.getname());
                    }
                    RefObject<VDS> tempRefObject = new RefObject<VDS>(selectedVds);
                    spmStatus =
                            handleSpmStatusResult(curVdsId, prioritizedVdsInPool, storagePool, tempRefObject, spmStatus);
                    selectedVds = tempRefObject.argvalue;

                    if (selectedVds != null) {
                        RefObject<VDS> tempRefObject2 = new RefObject<VDS>(selectedVds);
                        RefObject<SpmStatusResult> tempRefObject3 = new RefObject<SpmStatusResult>(spmStatus);
                        returnValue = handleSelectedVdsForSPM(storagePool, tempRefObject2, tempRefObject3, prevStatus);
                        selectedVds = tempRefObject2.argvalue;
                        spmStatus = tempRefObject3.argvalue;
                    } else {
                        mCurrentVdsId = null;
                    }
                    if (performedPoolConnect && selectedVds == null) {
                        // if could not start spm on this host and connected to
                        // pool here
                        // then disconnect
                        ResourceManager.getInstance().runVdsCommand(
                                VDSCommandType.DisconnectStoragePool,
                                new DisconnectStoragePoolVDSCommandParameters(selectedVdsId, _storagePoolId,
                                        selectedVdsSpmId));
                    }
                } else {

                    log.infoFormat("hostFromVds::selectedVds - {0}, spmStatus returned null!",
                            selectedVds.getName());
                    if (returnValueFromVds.getExceptionObject() instanceof IRSNoMasterDomainException) {
                        throw returnValueFromVds.getExceptionObject();
                    }
                }
            }
            return returnValue;
        }

        private List<VDS> getPrioritizedVdsInPool() {
            Guid curVdsId = (mCurrentVdsId != null) ? mCurrentVdsId : Guid.Empty;
            // Gets a list of the hosts in the storagePool, that are "UP", ordered
            // by vds_spm_priority (not including -1) and secondly ordered by RANDOM(), to
            // deal with the case that there are several hosts with the same priority.
            List<VDS> allVds = DbFacade.getInstance().getVdsDao().getListForSpmSelection(_storagePoolId);
            List<VDS> vdsRelevantForSpmSelection = new ArrayList<VDS>();
            for (VDS vds : allVds) {
                if (!mTriedVdssList.contains(vds.getId()) && !vds.getId().equals(curVdsId)) {
                    vdsRelevantForSpmSelection.add(vds);
                }
            }
            return vdsRelevantForSpmSelection;
        }

        private boolean _isSpmStartCalled;

        private String handleSelectedVdsForSPM(storage_pool storagePool, RefObject<VDS> selectedVds,
                                               RefObject<SpmStatusResult> spmStatus, StoragePoolStatus prevStatus) {
            String returnValue = null;
            if (spmStatus.argvalue == null || spmStatus.argvalue.getSpmStatus() != SpmStatus.SPM) {
                movePoolToProblematicInDB(storagePool, true);

                selectedVds.argvalue = null;
                log.infoFormat(
                        "spm start treatment ended and status is not SPM!!! status: {0} - setting selectedVds to null!",
                        spmStatus.argvalue.getSpmStatus().toString());
            } else {
                Init(selectedVds.argvalue);
                storagePool.setLVER(spmStatus.argvalue.getSpmLVER());
                storagePool.setspm_vds_id(selectedVds.argvalue.getId());
                // if were problemtaic or not operational and succeeded to find
                // host move pool to up
                if (prevStatus != StoragePoolStatus.NotOperational && prevStatus != StoragePoolStatus.Problematic) {
                    storagePool.setstatus(prevStatus);
                } else {
                    storagePool.setstatus(StoragePoolStatus.Up);
                }
                DbFacade.getInstance().getStoragePoolDao().update(storagePool);
                ResourceManager.getInstance()
                        .getEventListener()
                        .storagePoolStatusChanged(storagePool.getId(), storagePool.getstatus());

                returnValue = selectedVds.argvalue.getHostName();
                log.infoFormat("Initialize Irs proxy from vds: {0}", returnValue);
                AuditLogableBase logable = new AuditLogableBase(selectedVds.argvalue.getId());
                logable.addCustomValue("ServerIp", returnValue);
                AuditLogDirector.log(logable, AuditLogType.IRS_HOSTED_ON_VDS);
            }
            return returnValue;
        }

        /**
         * Waits for VDS if is initializing.
         *
         * @param curVdsId
         */
        private void waitForVdsIfIsInitializing(Guid curVdsId) {
            if (!Guid.Empty.equals(curVdsId)) {
                VDS vds = DbFacade.getInstance().getVdsDao().get(curVdsId);
                String vdsName = vds.getName();
                if (vds.getStatus() == VDSStatus.Initializing) {
                    final int DELAY = 5;// 5 Sec
                    int total = 0;
                    Integer maxSecToWait = Config.GetValue(ConfigValues.WaitForVdsInitInSec);
                    while (total <= maxSecToWait
                            && DbFacade.getInstance().getVdsDynamicDao().get(curVdsId).getstatus() == VDSStatus.Initializing) {
                        try {
                            Thread.sleep(DELAY * 1000);
                        } catch (InterruptedException e) {
                            log.errorFormat("Interrupt exception {0}", e.getMessage());
                            // exit the while block
                            break;
                        }
                        total += DELAY;
                        log.infoFormat("Waiting to Host {0} to finish initialization for {1} Sec.", vdsName, total);
                    }
                }
            }
        }

        private void movePoolToProblematicInDB(storage_pool storagePool, boolean resetSpmInDB) {
            ResourceManager
                    .getInstance()
                    .getEventListener()
                    .storagePoolStatusChange(storagePool.getId(), StoragePoolStatus.Problematic,
                            AuditLogType.SYSTEM_CHANGE_STORAGE_POOL_STATUS_PROBLEMATIC, VdcBllErrors.ENGINE);

            if (resetSpmInDB) {
                storagePool.setspm_vds_id(null);
                DbFacade.getInstance().getStoragePoolDao().update(storagePool);
            }
        }

        private SpmStatusResult handleSpmStatusResult(Guid curVdsId,
                                                      List<VDS> vdsByPool,
                                                      final storage_pool storagePool,
                                                      RefObject<VDS> selectedVds,
                                                      SpmStatusResult spmStatus) {
{'fraction_non_alphanumeric': 0.07993743311105622, 'fraction_numerical': 0.0015406508367733361, 'mean_word_length': 1.6432279523765116, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 132, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6547019', 'n_tokens_mistral': 21030, 'n_tokens_neox': 20101, 'n_words': 3731}