<commit_before>/*
 * Copyright (c) 2017 eilslabs.
 *
 * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt).
 */

package de.dkfz.roddy.config.loader

import de.dkfz.roddy.Constants
import de.dkfz.roddy.Roddy
import de.dkfz.roddy.StringConstants
import de.dkfz.roddy.client.RoddyStartupOptions
import de.dkfz.roddy.config.*
import de.dkfz.roddy.config.Configuration.ConfigurationType
import de.dkfz.roddy.config.converters.BashConverter
import de.dkfz.roddy.config.converters.YAMLConverter
import de.dkfz.roddy.core.Project
import de.dkfz.roddy.core.ProjectLoaderException
import de.dkfz.roddy.core.Workflow
import de.dkfz.roddy.knowledge.brawlworkflows.JBrawlWorkflow
import de.dkfz.roddy.knowledge.files.BaseFile
import de.dkfz.roddy.knowledge.files.FileObject
import de.dkfz.roddy.knowledge.files.FileStage
import de.dkfz.roddy.knowledge.nativeworkflows.NativeWorkflow
import de.dkfz.roddy.plugins.LibrariesFactory
import de.dkfz.roddy.plugins.PluginInfo
import de.dkfz.roddy.plugins.SyntheticPluginInfo

import de.dkfz.roddy.tools.LoggerWrapper
import de.dkfz.roddy.tools.RoddyConversionHelperMethods
import de.dkfz.roddy.tools.RoddyIOHelperMethods
import de.dkfz.roddy.tools.Tuple3
import groovy.transform.CompileStatic
import groovy.transform.TypeCheckingMode
import groovy.util.slurpersupport.NodeChild
import groovy.util.slurpersupport.NodeChildren
import groovy.xml.StreamingMarkupBuilder
import groovy.xml.XmlUtil
import org.apache.commons.io.filefilter.WildcardFileFilter
import org.xml.sax.SAXParseException

import java.lang.reflect.Field
import java.lang.reflect.Method
import java.lang.reflect.Modifier
import java.text.ParseException
import java.util.logging.Level

import static de.dkfz.roddy.StringConstants.*

/**
 * Factory for loading, importing, exporting and writing configuration files.
 * @author michael
 */
@groovy.transform.CompileStatic
class ConfigurationFactory {

    public static final String XMLTAG_EXECUTIONSERVICE_SSHUSER = "executionServiceSSHUser"

    public static final String XMLTAG_EXECUTIONSERVICE_SHOW_SSHCALLS = "executionServiceShowSSHCalls"

    public static final String XMLTAG_ATTRIBUTE_INHERITANALYSES = "inheritAnalyses"

    public static final String ERROR_PRINTOUT_XML_LINEPREFIX = "          "

    public static final LoggerWrapper logger = LoggerWrapper.getLogger(ConfigurationFactory.class.getSimpleName())


    private static ConfigurationFactory singleton

    private List<File> configurationDirectories = []

    private Map<String, PreloadedConfiguration> availableConfigurations = [:]

    private Map<ConfigurationType, List<PreloadedConfiguration>> availableConfigurationsByType = [:]

    private Map<ConfigurationType, Map<String, PreloadedConfiguration>> availableConfigurationsByTypeAndID = [:]


    static void initialize(List<File> configurationDirectories = null) {
        singleton = new ConfigurationFactory(configurationDirectories)
    }

    static ConfigurationFactory getInstance() {
        if (!singleton)
            initialize()
        return singleton
    }

    private ConfigurationFactory(List<File> configurationDirectories = null) {
        if (configurationDirectories == null)
            configurationDirectories = Roddy.getConfigurationDirectories()

        this.configurationDirectories.addAll(configurationDirectories)

        loadAvailableProjectConfigurationFiles()
    }

    private void loadAvailableProjectConfigurationFiles() {
        List<File> allFiles = []
        configurationDirectories.parallelStream().each {
            File baseDir ->
                logger.log(Level.CONFIG, "Searching for configuration files in: " + baseDir.toString())
                if (!baseDir.canRead() || !baseDir.canExecute()) {
                    logger.log(Level.SEVERE, "Cannot read from configuration directory ${baseDir.absolutePath}, does the folder exist und do you have access (read/execute) rights to it?")
                    throw new ConfigurationLoaderException("Cannot access (read and execute) configuration directory '${baseDir}'")
                }
                File[] files = baseDir.listFiles((FileFilter) new WildcardFileFilter(["*.xml", "*.sh", "*.yml"]))
                if (files == null) {
                    logger.info("No configuration files found in path ${baseDir.getAbsolutePath()}")
                }
                for (File f in files) {
                    synchronized (allFiles) {
                        if (!allFiles.contains(f))
                            allFiles << f
                    }
                }
        }

        Map<String, List<String>> pathsForCfgs = [:]
        List<String> duplicateConfigurationIDs = []
        for (File file in allFiles) {
            try {
                def icc = loadInformationalConfigurationContent(file)

                pathsForCfgs.get(icc.name, []) << icc.file.absolutePath
                if (availableConfigurations.containsKey(icc.name)) {
                    duplicateConfigurationIDs << icc.name
                    continue
                }

                availableConfigurations[icc.id] = icc
                availableConfigurationsByType.get(icc.type, []) << icc
                availableConfigurationsByTypeAndID.get(icc.type, [:])[icc.id] = icc
                for (PreloadedConfiguration iccSub in icc.getAllSubContent()) {
                    availableConfigurations[iccSub.id] = iccSub
                }

            } catch (ParseException ex) {
                logger.rare("File '${file}' is not a valid Roddy configuration file. Skipped!")
            } catch (UnknownConfigurationFileTypeException ex) {
                logger.severe("The file ${file.absolutePath} does not appear to be a valid Bash configuration file:\n\t ${ex.message}")
            } catch (Exception ex) {
                logger.severe("File ${file.absolutePath} cannot be loaded! Error in config file! ${ex.toString()}")
                logger.severe(RoddyIOHelperMethods.getStackTraceAsString(ex))
                throw new ConfigurationLoadError(null, null, "Could not load '${file}'", ex)
            }
        }
        if (duplicateConfigurationIDs) {

            StringBuilder messageForDuplicates = new StringBuilder("Configuration files using the same id were found:\n")
            for (String id in duplicateConfigurationIDs.sort().unique()) {
                messageForDuplicates << "\t" << id << ([" found in:"] << pathsForCfgs[id]).join("\n\t\t") << "\n"
            }
            messageForDuplicates << "\n" << (["This is not allowed! Check your configuration directories for files containing using same ids:"] + configurationDirectories.collect { it.absolutePath }).join("\n\t")

            throw new ConfigurationLoaderException(messageForDuplicates.toString())<commit_msg>Fixed previous commit :-(<commit_after>/*
 * Copyright (c) 2017 eilslabs.
 *
 * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt).
 */

package de.dkfz.roddy.config.loader

import de.dkfz.roddy.Constants
import de.dkfz.roddy.Roddy
import de.dkfz.roddy.StringConstants
import de.dkfz.roddy.client.RoddyStartupOptions
import de.dkfz.roddy.config.*
import de.dkfz.roddy.config.Configuration.ConfigurationType
import de.dkfz.roddy.config.converters.BashConverter
import de.dkfz.roddy.config.converters.YAMLConverter
import de.dkfz.roddy.core.Project
import de.dkfz.roddy.core.ProjectLoaderException
import de.dkfz.roddy.core.Workflow
import de.dkfz.roddy.knowledge.brawlworkflows.JBrawlWorkflow
import de.dkfz.roddy.knowledge.files.BaseFile
import de.dkfz.roddy.knowledge.files.FileObject
import de.dkfz.roddy.knowledge.files.FileStage
import de.dkfz.roddy.knowledge.nativeworkflows.NativeWorkflow
import de.dkfz.roddy.plugins.LibrariesFactory
import de.dkfz.roddy.plugins.PluginInfo
import de.dkfz.roddy.plugins.SyntheticPluginInfo

import de.dkfz.roddy.tools.LoggerWrapper
import de.dkfz.roddy.tools.RoddyConversionHelperMethods
import de.dkfz.roddy.tools.RoddyIOHelperMethods
import de.dkfz.roddy.tools.Tuple3
import groovy.transform.CompileStatic
import groovy.transform.TypeCheckingMode
import groovy.util.slurpersupport.NodeChild
import groovy.util.slurpersupport.NodeChildren
import groovy.xml.StreamingMarkupBuilder
import groovy.xml.XmlUtil
import org.apache.commons.io.filefilter.WildcardFileFilter
import org.xml.sax.SAXParseException

import java.lang.reflect.Field
import java.lang.reflect.Method
import java.lang.reflect.Modifier
import java.text.ParseException
import java.util.logging.Level

import static de.dkfz.roddy.StringConstants.*

/**
 * Factory for loading, importing, exporting and writing configuration files.
 * @author michael
 */
@groovy.transform.CompileStatic
class ConfigurationFactory {

    public static final String XMLTAG_EXECUTIONSERVICE_SSHUSER = "executionServiceSSHUser"

    public static final String XMLTAG_EXECUTIONSERVICE_SHOW_SSHCALLS = "executionServiceShowSSHCalls"

    public static final String XMLTAG_ATTRIBUTE_INHERITANALYSES = "inheritAnalyses"

    public static final String ERROR_PRINTOUT_XML_LINEPREFIX = "          "

    public static final LoggerWrapper logger = LoggerWrapper.getLogger(ConfigurationFactory.class.getSimpleName())


    private static ConfigurationFactory singleton

    private List<File> configurationDirectories = []

    private Map<String, PreloadedConfiguration> availableConfigurations = [:]

    private Map<ConfigurationType, List<PreloadedConfiguration>> availableConfigurationsByType = [:]

    private Map<ConfigurationType, Map<String, PreloadedConfiguration>> availableConfigurationsByTypeAndID = [:]


    static void initialize(List<File> configurationDirectories = null) {
        singleton = new ConfigurationFactory(configurationDirectories)
    }

    static ConfigurationFactory getInstance() {
        if (!singleton)
            initialize()
        return singleton
    }

    private ConfigurationFactory(List<File> configurationDirectories = null) {
        if (configurationDirectories == null)
            configurationDirectories = Roddy.getConfigurationDirectories()

        this.configurationDirectories.addAll(configurationDirectories)

        loadAvailableProjectConfigurationFiles()
    }

    private void loadAvailableProjectConfigurationFiles() {
        List<File> allFiles = []
        configurationDirectories.parallelStream().each {
            File baseDir ->
                logger.log(Level.CONFIG, "Searching for configuration files in: " + baseDir.toString())
                if (!baseDir.canRead() || !baseDir.canExecute()) {
                    logger.log(Level.SEVERE, "Cannot read from configuration directory ${baseDir.absolutePath}, does the folder exist und do you have access (read/execute) rights to it?")
                    throw new ConfigurationLoaderException("Cannot access (read and execute) configuration directory '${baseDir}'")
                }
                File[] files = baseDir.listFiles((FileFilter) new WildcardFileFilter(["*.xml", "*.sh", "*.yml"]))
                if (files == null) {
                    logger.info("No configuration files found in path ${baseDir.getAbsolutePath()}")
                }
                for (File f in files) {
                    synchronized (allFiles) {
                        if (!allFiles.contains(f))
                            allFiles << f
                    }
                }
        }

        Map<String, List<String>> pathsForCfgs = [:]
        List<String> duplicateConfigurationIDs = []
        for (File file in allFiles) {
            try {
                def icc = loadInformationalConfigurationContent(file)

                pathsForCfgs.get(icc.name, []) << icc.file.absolutePath
                if (availableConfigurations.containsKey(icc.name)) {
                    duplicateConfigurationIDs << icc.name
                    continue
                }

                availableConfigurations[icc.id] = icc
                availableConfigurationsByType.get(icc.type, []) << icc
                availableConfigurationsByTypeAndID.get(icc.type, [:])[icc.id] = icc
                for (PreloadedConfiguration iccSub in icc.getAllSubContent()) {
                    availableConfigurations[iccSub.id] = iccSub
                }

            } catch (ParseException ex) {
                logger.rare("File '${file}' is not a valid Roddy configuration file. Skipped!")
            } catch (UnknownConfigurationFileTypeException ex) {
                logger.severe("The file ${file.absolutePath} does not appear to be a valid Bash configuration file:\n\t ${ex.message}")
            } catch (Exception ex) {
                logger.severe("File ${file.absolutePath} cannot be loaded! Error in config file! ${ex.toString()}")
                logger.severe(RoddyIOHelperMethods.getStackTraceAsString(ex))
                throw new ConfigurationLoadError(null, null, "Could not load '${file}'", ex)
            }
        }
        if (duplicateConfigurationIDs) {

            StringBuilder messageForDuplicates = new StringBuilder("Configuration files using the same id were found:\n")
            for (String id in duplicateConfigurationIDs.sort().unique()) {
                messageForDuplicates << "\t" << id << ([" found in:"] + pathsForCfgs[id]).join("\n\t\t") << "\n"
            }
            messageForDuplicates << "\n" << (["This is not allowed! Check your configuration directories for files containing using same ids:"] + configurationDirectories.collect { it.absolutePath }).join("\n\t")

            throw new ConfigurationLoaderException(messageForDuplicates.toString())
{'fraction_non_alphanumeric': 0.09413140393188628, 'fraction_numerical': 0.0007308338814587444, 'mean_word_length': 3.076258564194221, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 29, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6817122', 'n_tokens_mistral': 3843, 'n_tokens_neox': 3678, 'n_words': 792}