<commit_before>/*
 * Copyright (c) 2008-2011 by <NAME>,
 *               Zuse Institute Berlin
 *
 * Licensed under the BSD License, see LICENSE file for details.
 *
 */

package org.xtreemfs.mrc.utils;

import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.xtreemfs.common.ReplicaUpdatePolicies;
import org.xtreemfs.common.uuids.ServiceUUID;
import org.xtreemfs.common.uuids.UnknownUUIDException;
import org.xtreemfs.common.xloc.ReplicationFlags;
import org.xtreemfs.foundation.json.JSONException;
import org.xtreemfs.foundation.json.JSONParser;
import org.xtreemfs.foundation.logging.Logging;
import org.xtreemfs.foundation.logging.Logging.Category;
import org.xtreemfs.foundation.pbrpc.generatedinterfaces.RPC.POSIXErrno;
import org.xtreemfs.foundation.util.OutputUtils;
import org.xtreemfs.mrc.MRCConfig;
import org.xtreemfs.mrc.MRCException;
import org.xtreemfs.mrc.UserException;
import org.xtreemfs.mrc.ac.FileAccessManager;
import org.xtreemfs.mrc.database.AtomicDBUpdate;
import org.xtreemfs.mrc.database.DatabaseException;
import org.xtreemfs.mrc.database.DatabaseResultSet;
import org.xtreemfs.mrc.database.StorageManager;
import org.xtreemfs.mrc.database.VolumeInfo;
import org.xtreemfs.mrc.database.VolumeManager;
import org.xtreemfs.mrc.metadata.FileMetadata;
import org.xtreemfs.mrc.metadata.ReplicationPolicy;
import org.xtreemfs.mrc.metadata.StripingPolicy;
import org.xtreemfs.mrc.metadata.XAttr;
import org.xtreemfs.mrc.metadata.XLoc;
import org.xtreemfs.mrc.metadata.XLocList;
import org.xtreemfs.mrc.osdselection.OSDStatusManager;
import org.xtreemfs.pbrpc.generatedinterfaces.DIR.Service;
import org.xtreemfs.pbrpc.generatedinterfaces.DIR.ServiceDataMap;
import org.xtreemfs.pbrpc.generatedinterfaces.DIR.ServiceSet;
import org.xtreemfs.pbrpc.generatedinterfaces.DIR.ServiceType;
import org.xtreemfs.pbrpc.generatedinterfaces.GlobalTypes.KeyValuePair;
import org.xtreemfs.pbrpc.generatedinterfaces.GlobalTypes.VivaldiCoordinates;

public class MRCHelper {
    
    public static class GlobalFileIdResolver {
        
        final String volumeId;
        
        final long   localFileId;
        
        public GlobalFileIdResolver(String globalFileId) throws UserException {
            
            try {
                int i = globalFileId.indexOf(':');
                volumeId = globalFileId.substring(0, i);
                localFileId = Long.parseLong(globalFileId.substring(i + 1));
            } catch (Exception exc) {
                throw new UserException(POSIXErrno.POSIX_ERROR_EINVAL, "invalid global file ID: " + globalFileId
                        + "; expected pattern: <volume_ID>:<local_file_ID>");
            }
        }
        
        public String getVolumeId() {
            return volumeId;
        }
        
        public long getLocalFileId() {
            return localFileId;
        }
    }
    
    public static final String POLICY_ATTR_PREFIX = "policies";
    
    public static final String VOL_ATTR_PREFIX    = "volattr";
    
    public static final String XTREEMFS_POLICY_ATTR_PREFIX = "xtreemfs." + POLICY_ATTR_PREFIX + ".";
    
    public enum SysAttrs {
            locations,
            file_id,
            object_type,
            url,
            owner,
            group,
            default_sp,
            ac_policy_id,
            rsel_policy,
            osel_policy,
            usable_osds,
            free_space,
            used_space,
            num_files,
            num_dirs,
            snapshots,
            snapshots_enabled,
            snapshot_time,
            acl,
            read_only,
            mark_replica_complete,
            set_repl_update_policy,
            default_rp,
            quota
    }
    
    public enum FileType {
        nexists, dir, file
    }
    
    public static Service createDSVolumeInfo(VolumeInfo vol, OSDStatusManager osdMan, StorageManager sMan,
            String mrcUUID) {
        
        String free = String.valueOf(osdMan.getFreeSpace(vol.getId()));
        String volSize = null;
        try {
            volSize = String.valueOf(sMan.getVolumeInfo().getVolumeSize());
        } catch (DatabaseException e) {
            Logging.logMessage(Logging.LEVEL_WARN, Category.storage, null,
                    "could not retrieve volume size from database for volume '%s': %s",
                    new Object[] { vol.getName(), e.toString() });
        }
        
        ServiceDataMap.Builder dmap = buildServiceDataMap("mrc", mrcUUID, "free", free, "used", volSize);
        
        try {
            
            DatabaseResultSet<XAttr> attrIt = sMan.getXAttrs(1, StorageManager.SYSTEM_UID);
            while (attrIt.hasNext()) {
                XAttr attr = attrIt.next();
                if (attr.getKey().startsWith("xtreemfs.volattr.")) {
                    byte[] value = attr.getValue();
                    dmap.addData(KeyValuePair.newBuilder()
                            .setKey("attr." + attr.getKey().substring("xtreemfs.volattr.".length()))
                            .setValue(value == null ? null : new String(value)));
                }
            }
            attrIt.destroy();
            
        } catch (DatabaseException e) {
            Logging.logMessage(Logging.LEVEL_ERROR, Category.storage, null, OutputUtils.stackTraceToString(e),
                    new Object[0]);
        }
        <commit_msg>servers: Consider volume quota in available space computation (DIR status page).<commit_after>/*
 * Copyright (c) 2008-2011 by <NAME>,
 *               Zuse Institute Berlin
 *
 * Licensed under the BSD License, see LICENSE file for details.
 *
 */

package org.xtreemfs.mrc.utils;

import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.xtreemfs.common.ReplicaUpdatePolicies;
import org.xtreemfs.common.uuids.ServiceUUID;
import org.xtreemfs.common.uuids.UnknownUUIDException;
import org.xtreemfs.common.xloc.ReplicationFlags;
import org.xtreemfs.foundation.json.JSONException;
import org.xtreemfs.foundation.json.JSONParser;
import org.xtreemfs.foundation.logging.Logging;
import org.xtreemfs.foundation.logging.Logging.Category;
import org.xtreemfs.foundation.pbrpc.generatedinterfaces.RPC.POSIXErrno;
import org.xtreemfs.foundation.util.OutputUtils;
import org.xtreemfs.mrc.MRCConfig;
import org.xtreemfs.mrc.MRCException;
import org.xtreemfs.mrc.UserException;
import org.xtreemfs.mrc.ac.FileAccessManager;
import org.xtreemfs.mrc.database.AtomicDBUpdate;
import org.xtreemfs.mrc.database.DatabaseException;
import org.xtreemfs.mrc.database.DatabaseResultSet;
import org.xtreemfs.mrc.database.StorageManager;
import org.xtreemfs.mrc.database.VolumeInfo;
import org.xtreemfs.mrc.database.VolumeManager;
import org.xtreemfs.mrc.metadata.FileMetadata;
import org.xtreemfs.mrc.metadata.ReplicationPolicy;
import org.xtreemfs.mrc.metadata.StripingPolicy;
import org.xtreemfs.mrc.metadata.XAttr;
import org.xtreemfs.mrc.metadata.XLoc;
import org.xtreemfs.mrc.metadata.XLocList;
import org.xtreemfs.mrc.osdselection.OSDStatusManager;
import org.xtreemfs.pbrpc.generatedinterfaces.DIR.Service;
import org.xtreemfs.pbrpc.generatedinterfaces.DIR.ServiceDataMap;
import org.xtreemfs.pbrpc.generatedinterfaces.DIR.ServiceSet;
import org.xtreemfs.pbrpc.generatedinterfaces.DIR.ServiceType;
import org.xtreemfs.pbrpc.generatedinterfaces.GlobalTypes.KeyValuePair;
import org.xtreemfs.pbrpc.generatedinterfaces.GlobalTypes.VivaldiCoordinates;

public class MRCHelper {
    
    public static class GlobalFileIdResolver {
        
        final String volumeId;
        
        final long   localFileId;
        
        public GlobalFileIdResolver(String globalFileId) throws UserException {
            
            try {
                int i = globalFileId.indexOf(':');
                volumeId = globalFileId.substring(0, i);
                localFileId = Long.parseLong(globalFileId.substring(i + 1));
            } catch (Exception exc) {
                throw new UserException(POSIXErrno.POSIX_ERROR_EINVAL, "invalid global file ID: " + globalFileId
                        + "; expected pattern: <volume_ID>:<local_file_ID>");
            }
        }
        
        public String getVolumeId() {
            return volumeId;
        }
        
        public long getLocalFileId() {
            return localFileId;
        }
    }
    
    public static final String POLICY_ATTR_PREFIX = "policies";
    
    public static final String VOL_ATTR_PREFIX    = "volattr";
    
    public static final String XTREEMFS_POLICY_ATTR_PREFIX = "xtreemfs." + POLICY_ATTR_PREFIX + ".";
    
    public enum SysAttrs {
            locations,
            file_id,
            object_type,
            url,
            owner,
            group,
            default_sp,
            ac_policy_id,
            rsel_policy,
            osel_policy,
            usable_osds,
            free_space,
            used_space,
            num_files,
            num_dirs,
            snapshots,
            snapshots_enabled,
            snapshot_time,
            acl,
            read_only,
            mark_replica_complete,
            set_repl_update_policy,
            default_rp,
            quota
    }
    
    public enum FileType {
        nexists, dir, file
    }
    
    public static Service createDSVolumeInfo(VolumeInfo vol, OSDStatusManager osdMan, StorageManager sMan,
            String mrcUUID) {
        
        String free = String.valueOf(osdMan.getFreeSpace(vol.getId()));
        String volSize = null;
        try {
            volSize = String.valueOf(sMan.getVolumeInfo().getVolumeSize());
            try {
                // Use minimum of free space relative to the quota and free space on osds as free disk space.
                long quota = vol.getVolumeQuota();
                long quotaFreeSpace = quota - vol.getVolumeSize();
                if (quota != 0 && quotaFreeSpace < Long.valueOf(free)) {
                    quotaFreeSpace = quotaFreeSpace < 0 ? 0 : quotaFreeSpace;
                    free = String.valueOf(quotaFreeSpace);
                }
            } catch (DatabaseException e) {
                Logging.logMessage(Logging.LEVEL_WARN, Category.storage, null,
                        "could not retrieve volume quota from database for volume '%s': %s",
                        new Object[] { vol.getName(), e.toString() });
            }
        } catch (DatabaseException e) {
            Logging.logMessage(Logging.LEVEL_WARN, Category.storage, null,
                    "could not retrieve volume size from database for volume '%s': %s",
                    new Object[] { vol.getName(), e.toString() });
        }
        
        ServiceDataMap.Builder dmap = buildServiceDataMap("mrc", mrcUUID, "free", free, "used", volSize);
        
        try {
            
            DatabaseResultSet<XAttr> attrIt = sMan.getXAttrs(1, StorageManager.SYSTEM_UID);
            while (attrIt.hasNext()) {
                XAttr attr = attrIt.next();
                if (attr.getKey().startsWith("xtreemfs.volattr.")) {
                    byte[] value = attr.getValue();
                    dmap.addData(KeyValuePair.newBuilder()
                            .setKey("attr." + attr.getKey().substring("xtreemfs.volattr.".length()))
                            .setValue(value == null ? null : new String(value)));
                }
            }
            attrIt.destroy();
            
        } catch (DatabaseException e) {
            Logging.logMessage(Logging.LEVEL_ERROR, Category.storage, null, OutputUtils.stackTraceToString(e),
                    new Object[0]);
        }
        
{'fraction_non_alphanumeric': 0.0929429182431321, 'fraction_numerical': 0.0022207599934199705, 'mean_word_length': 2.3094719651605877, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5932158', 'n_tokens_mistral': 3636, 'n_tokens_neox': 3584, 'n_words': 663}