<commit_before>//----------------------------------------------------------------------
//This code is developed as part of the Java CoG Kit project
//The terms of the license can be found at http://www.cogkit.org/license
//This message may not be removed or altered.
//----------------------------------------------------------------------

/*
 * Created on Jul 5, 2014
 */
package org.griphyn.vdl.util;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.log4j.Logger;
import org.globus.cog.abstraction.impl.common.AbstractionFactory;
import org.globus.cog.abstraction.impl.common.ProviderMethodException;
import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl;
import org.globus.cog.abstraction.impl.common.task.InvalidProviderException;
import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl;
import org.globus.cog.abstraction.impl.common.task.ServiceImpl;
import org.globus.cog.abstraction.interfaces.ExecutionService;
import org.globus.cog.abstraction.interfaces.Service;
import org.globus.cog.abstraction.interfaces.ServiceContact;
import org.globus.cog.karajan.util.BoundContact;
import org.globus.swift.catalog.site.Application;
import org.globus.swift.catalog.site.SwiftContact;
import org.globus.swift.catalog.site.SwiftContactSet;
import org.griphyn.vdl.util.ConfigTree.Node;

import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import com.typesafe.config.ConfigIncludeContext;
import com.typesafe.config.ConfigIncluder;
import com.typesafe.config.ConfigObject;
import com.typesafe.config.ConfigParseOptions;
import com.typesafe.config.ConfigSyntax;
import com.typesafe.config.ConfigValue;
import com.typesafe.config.ConfigValueType;

public class SwiftConfig implements Cloneable {
    public static final Logger logger = Logger.getLogger(SwiftConfig.class);
    
    public static final boolean CHECK_DYNAMIC_NAMES = true;
    public static final boolean BUILD_CHECK = true;
    
    public static final List<String> DEFAULT_LOCATIONS;
    
    public enum Key {
        DM_CHECKER("mappingCheckerEnabled"),
        PROVENANCE_LOG("logProvenance"),
        FILE_GC_ENABLED("fileGCEnabled"),
        TICKER_ENABLED("tickerEnabled"),
        TICKER_DATE_FORMAT("tickerDateFormat"),
        TICKER_PREFIX("tickerPrefix"),
        TRACING_ENABLED("tracingEnabled"),
        FOREACH_MAX_THREADS("maxForeachThreads"),
        CACHING_ALGORITHM("cachingAlgorithm"), 
        REPLICATION_ENABLED("replicationEnabled"), 
        WRAPPER_STAGING_LOCAL_SERVER("wrapperStagingLocalServer"), 
        REPLICATION_MIN_QUEUE_TIME("replicationMinQueueTime"), 
        REPLICATION_LIMIT("replicationLimit"), 
        WRAPPER_INVOCATION_MODE("wrapperInvocationMode"), 
        CDM_BROADCAST_MODE("CDMBroadcastMode"), 
        CMD_LOG_FILE("CDMLogFile");
        
        public String propName;
        private Key(String propName) {
            this.propName = propName;
        }
    }
    
    public static SwiftConfigSchema SCHEMA;
    
    static {
        SCHEMA = new SwiftConfigSchema();
        DEFAULT_LOCATIONS = new ArrayList<String>();
        if (System.getenv("SWIFT_SITE_CONF") != null) {
            DEFAULT_LOCATIONS.add(System.getenv("SWIFT_SITE_CONF"));
        }
        String swiftHome = System.getProperty("swift.home");
        if (swiftHome == null) {
            swiftHome = System.getProperty("swift.home");
            if (swiftHome == null) {
                throw new IllegalStateException("swift.home is not set");
            }
        }
        
        DEFAULT_LOCATIONS.add(swiftHome + File.separator + 
            "etc" + File.separator + "swift.conf");
        
        // check keys against schema
        
        for (Key k : Key.values()) {
            if (!SCHEMA.isNameValid(k.propName)) {
                throw new IllegalArgumentException("Invalid property name for config key '" + k + "': " + k.propName);
            }
        }
    }
    
    private static class KVPair {
        public final String key;
        public final String value;
        
        public KVPair(String key, String value) {
            this.key = key;
            this.value = value;
        }
    }
    
    private static class IncluderWrapper implements ConfigIncluder {
        private final ConfigIncluder d;
        
        public IncluderWrapper(ConfigIncluder d) {
            this.d = d;
        }

        @Override
        public ConfigIncluder withFallback(ConfigIncluder fallback) {
            return this;
        }

        @Override
        public ConfigObject include(ConfigIncludeContext context, String what) {
            int b = what.indexOf("${");
            while (b != -1) {
                int e = what.indexOf("}", b);
                String var = what.substring(b + 2, e);
                what = what.substring(0, b) + resolve(var) + what.substring(e + 1);
                b = what.indexOf("${");
            }
            return ConfigFactory.parseFile(new File(what)).root();
        }

        private String resolve(String var) {
            String v = null;
            if (var.startsWith("env.")) {
                v = System.getenv(var.substring(4));
            }
            else {
                v = System.getProperty(var);
            }
            if (v == null) {
                throw new IllegalArgumentException("No such system property or environment variable: '" + var + "'");
            }
            return v;
        }
    }
    
    public static SwiftConfig load(String fileName) {
        return load(fileName, null);
    }

    public static SwiftConfig load(String fileName, Map<String, Object> override) {
        logger.info("Loading swift configuration file: " + fileName);
        ConfigParseOptions opt = ConfigParseOptions.defaults();
        opt = opt.setIncluder(new IncluderWrapper(opt.getIncluder())).
            setSyntax(ConfigSyntax.CONF).setAllowMissing(false);
        Config conf = ConfigFactory.parseFile(new File(fileName), opt);
        Config oconf = ConfigFactory.parseMap(override, "<command line>");
        conf = oconf.withFallback(conf);
        conf = conf.resolveWith(getSubstitutions());
        ConfigTree<Object> out = SCHEMA.validate(conf);
        SwiftConfig sc = new SwiftConfig();
        sc.setFileName(fileName);
        sc.build(out);<commit_msg>check for null command line overrides<commit_after>//----------------------------------------------------------------------
//This code is developed as part of the Java CoG Kit project
//The terms of the license can be found at http://www.cogkit.org/license
//This message may not be removed or altered.
//----------------------------------------------------------------------

/*
 * Created on Jul 5, 2014
 */
package org.griphyn.vdl.util;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.log4j.Logger;
import org.globus.cog.abstraction.impl.common.AbstractionFactory;
import org.globus.cog.abstraction.impl.common.ProviderMethodException;
import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl;
import org.globus.cog.abstraction.impl.common.task.InvalidProviderException;
import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl;
import org.globus.cog.abstraction.impl.common.task.ServiceImpl;
import org.globus.cog.abstraction.interfaces.ExecutionService;
import org.globus.cog.abstraction.interfaces.Service;
import org.globus.cog.abstraction.interfaces.ServiceContact;
import org.globus.cog.karajan.util.BoundContact;
import org.globus.swift.catalog.site.Application;
import org.globus.swift.catalog.site.SwiftContact;
import org.globus.swift.catalog.site.SwiftContactSet;
import org.griphyn.vdl.util.ConfigTree.Node;

import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import com.typesafe.config.ConfigIncludeContext;
import com.typesafe.config.ConfigIncluder;
import com.typesafe.config.ConfigObject;
import com.typesafe.config.ConfigParseOptions;
import com.typesafe.config.ConfigSyntax;
import com.typesafe.config.ConfigValue;
import com.typesafe.config.ConfigValueType;

public class SwiftConfig implements Cloneable {
    public static final Logger logger = Logger.getLogger(SwiftConfig.class);
    
    public static final boolean CHECK_DYNAMIC_NAMES = true;
    public static final boolean BUILD_CHECK = true;
    
    public static final List<String> DEFAULT_LOCATIONS;
    
    public enum Key {
        DM_CHECKER("mappingCheckerEnabled"),
        PROVENANCE_LOG("logProvenance"),
        FILE_GC_ENABLED("fileGCEnabled"),
        TICKER_ENABLED("tickerEnabled"),
        TICKER_DATE_FORMAT("tickerDateFormat"),
        TICKER_PREFIX("tickerPrefix"),
        TRACING_ENABLED("tracingEnabled"),
        FOREACH_MAX_THREADS("maxForeachThreads"),
        CACHING_ALGORITHM("cachingAlgorithm"), 
        REPLICATION_ENABLED("replicationEnabled"), 
        WRAPPER_STAGING_LOCAL_SERVER("wrapperStagingLocalServer"), 
        REPLICATION_MIN_QUEUE_TIME("replicationMinQueueTime"), 
        REPLICATION_LIMIT("replicationLimit"), 
        WRAPPER_INVOCATION_MODE("wrapperInvocationMode"), 
        CDM_BROADCAST_MODE("CDMBroadcastMode"), 
        CMD_LOG_FILE("CDMLogFile");
        
        public String propName;
        private Key(String propName) {
            this.propName = propName;
        }
    }
    
    public static SwiftConfigSchema SCHEMA;
    
    static {
        SCHEMA = new SwiftConfigSchema();
        DEFAULT_LOCATIONS = new ArrayList<String>();
        if (System.getenv("SWIFT_SITE_CONF") != null) {
            DEFAULT_LOCATIONS.add(System.getenv("SWIFT_SITE_CONF"));
        }
        String swiftHome = System.getProperty("swift.home");
        if (swiftHome == null) {
            swiftHome = System.getProperty("swift.home");
            if (swiftHome == null) {
                throw new IllegalStateException("swift.home is not set");
            }
        }
        
        DEFAULT_LOCATIONS.add(swiftHome + File.separator + 
            "etc" + File.separator + "swift.conf");
        
        // check keys against schema
        
        for (Key k : Key.values()) {
            if (!SCHEMA.isNameValid(k.propName)) {
                throw new IllegalArgumentException("Invalid property name for config key '" + k + "': " + k.propName);
            }
        }
    }
    
    private static class KVPair {
        public final String key;
        public final String value;
        
        public KVPair(String key, String value) {
            this.key = key;
            this.value = value;
        }
    }
    
    private static class IncluderWrapper implements ConfigIncluder {
        private final ConfigIncluder d;
        
        public IncluderWrapper(ConfigIncluder d) {
            this.d = d;
        }

        @Override
        public ConfigIncluder withFallback(ConfigIncluder fallback) {
            return this;
        }

        @Override
        public ConfigObject include(ConfigIncludeContext context, String what) {
            int b = what.indexOf("${");
            while (b != -1) {
                int e = what.indexOf("}", b);
                String var = what.substring(b + 2, e);
                what = what.substring(0, b) + resolve(var) + what.substring(e + 1);
                b = what.indexOf("${");
            }
            return ConfigFactory.parseFile(new File(what)).root();
        }

        private String resolve(String var) {
            String v = null;
            if (var.startsWith("env.")) {
                v = System.getenv(var.substring(4));
            }
            else {
                v = System.getProperty(var);
            }
            if (v == null) {
                throw new IllegalArgumentException("No such system property or environment variable: '" + var + "'");
            }
            return v;
        }
    }
    
    public static SwiftConfig load(String fileName) {
        return load(fileName, null);
    }

    public static SwiftConfig load(String fileName, Map<String, Object> override) {
        logger.info("Loading swift configuration file: " + fileName);
        ConfigParseOptions opt = ConfigParseOptions.defaults();
        opt = opt.setIncluder(new IncluderWrapper(opt.getIncluder())).
            setSyntax(ConfigSyntax.CONF).setAllowMissing(false);
        Config conf = ConfigFactory.parseFile(new File(fileName), opt);
        if (override != null) {
            Config oconf = ConfigFactory.parseMap(override, "<command line>");
            conf = oconf.withFallback(conf);
        }
        conf = conf.resolveWith(getSubstitutions());
        ConfigTree<Object> out = SCHEMA.validate(conf);
        SwiftConfig sc = new SwiftConfig();
        sc.setFileName(fileName);
        sc.build(out);
{'fraction_non_alphanumeric': 0.12168949771689498, 'fraction_numerical': 0.0016742770167427702, 'mean_word_length': 2.4418543740178102, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3388183', 'n_tokens_mistral': 3775, 'n_tokens_neox': 3662, 'n_words': 753}