<commit_before>/*
 * Copyright (c) 2017 eilslabs.
 *
 * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt).
 */

package de.dkfz.roddy.execution.jobs

import de.dkfz.roddy.AvailableFeatureToggles
import de.dkfz.roddy.Constants
import de.dkfz.roddy.Roddy
import de.dkfz.roddy.StringConstants
import de.dkfz.roddy.config.*
import de.dkfz.roddy.config.converters.BashConverter
import de.dkfz.roddy.core.ExecutionContext
import de.dkfz.roddy.core.ExecutionContextError
import de.dkfz.roddy.core.ExecutionContextLevel
import de.dkfz.roddy.execution.io.fs.FileSystemAccessProvider
import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager
import de.dkfz.roddy.execution.jobs.direct.synchronousexecution.DirectSynchronousExecutionJobManager
import de.dkfz.roddy.knowledge.files.BaseFile
import de.dkfz.roddy.knowledge.files.FileGroup
import de.dkfz.roddy.tools.LoggerWrapper
import de.dkfz.roddy.tools.RoddyIOHelperMethods
import groovy.transform.CompileDynamic

import static de.dkfz.roddy.Constants.NO_VALUE
import static de.dkfz.roddy.config.FilenamePattern.PLACEHOLDER_JOBPARAMETER

@groovy.transform.CompileStatic
class Job extends BEJob<BEJob, BEJobResult> {

    private static final de.dkfz.roddy.tools.LoggerWrapper logger = de.dkfz.roddy.tools.LoggerWrapper.getLogger(BEJob.class.getSimpleName())

    public static final String TOOLID_WRAPIN_SCRIPT = "wrapinScript"
    public static final String PARM_WRAPPED_SCRIPT = "WRAPPED_SCRIPT"
    public static final String PARM_WRAPPED_SCRIPT_MD5 = "WRAPPED_SCRIPT_MD5"

    /**
     * The current context.
     */
    public final ExecutionContext context

    /**
     * The local tool path
     */
    private final File localToolPath

    /**
     * The tool you want to call.
     */
    private final String toolID

    /**
     * Keeps a list of all unchanged, initial parameters, including default job parameters.
     */
    protected final Map<String, Object> allRawInputParameters

    /**
     * Provide a list of files if you want to generate job dependencies.
     */
    public transient final List<BaseFile> parentFiles

    private Map<String, Object> initialInputParameters = [:]

    /**
     * Command object of last execution.
     */
    protected transient Command lastCommand

    /**
     * The list contains files which should be validated for a job restart.
     */
    public final List<BaseFile> filesToVerify

    Job(ExecutionContext context, String jobName, String toolID, List<String> arrayIndices, Map<String, Object> parameters, List<BaseFile> parentFiles) {
        this(context, jobName, toolID, arrayIndices, parameters, parentFiles, null)
    }

    Job(ExecutionContext context, String jobName, String toolID, Map<String, Object> parameters, List<BaseFile> parentFiles, List<BaseFile> filesToVerify) {
        this(context, jobName, toolID, null, parameters, parentFiles, filesToVerify)
    }

    Job(ExecutionContext context, String jobName, String toolID, Map<String, Object> parameters, List<BaseFile> parentFiles) {
        this(context, jobName, toolID, null, parameters, parentFiles, null)
    }

    Job(ExecutionContext context, String jobName, String toolID, List<String> arrayIndices, List<BaseFile> filesToVerify, Map<String, Object> parameters) {
        this(context, jobName, toolID, arrayIndices, parameters, null, filesToVerify)
    }

    Job(ExecutionContext context, String jobName, List<BaseFile> filesToVerify, String toolID, Map<String, Object> parameters) {
        this(context, jobName, toolID, null, parameters, null, filesToVerify)
    }

    Job(ExecutionContext context, String jobName, String toolID, List<String> arrayIndices, Map<String, Object> parameters) {
        this(context, jobName, toolID, arrayIndices, parameters, null, null)
    }

    Job(ExecutionContext context, String jobName, String toolID, Map<String, Object> parameters) {
        this(context, jobName, toolID, null, parameters, null, null)
    }

//    /**
//     * This is for job implementations which do the writeConfigurationFile on their own.
//     */
//    protected BEJob(String jobName, ExecutionContext context, String toolID, Map<String, String> parameters, List<String> arrayIndices, List<BaseFile> parentFiles, List<BaseFile> filesToVerify) {
//        super(jobName
//                , context.getConfiguration().getProcessingToolPath(context, toolID)
//                , toolID != null && toolID.trim().length() > 0 ? context.getConfiguration().getProcessingToolMD5(toolID): null
//                , arrayIndices
//                , parameters
//                , collectParentJobsFromFiles(parentFiles)
//                , collectDependencyIDsFromFiles(parentFiles)
//                , Roddy.getJobManager())
//        this.toolID = toolID
//        this.context = context
//        this.parentFiles = parentFiles
//        this.filesToVerify = filesToVerify
//        this.context.addExecutedJob(this)
//    }

    Job(ExecutionContext context, String jobName, String toolID, List<String> arrayIndices, Map<String, Object> inputParameters, List<BaseFile> parentFiles, List<BaseFile> filesToVerify) {
        this(context, jobName, toolID, null, arrayIndices, inputParameters, parentFiles, filesToVerify)
    }

    private static List<BEJobID> jobs2jobIDs(List<BEJob> jobs) {
        if (null == jobs) {
            return new LinkedList<BEJobID>()
        } else {
            return jobs.collect { it.runResult.jobID }
        }
    }

    private static List<BEJob> reconcileParentJobInformation(List<BEJob> parentJobs, List<BEJobID> parentJobIDs, BatchEuphoriaJobManager jobManager) {
        List<BEJob> pJobs
        if ((null != parentJobIDs && !parentJobIDs.isEmpty()) &&
                (null != parentJobs && !parentJobs.isEmpty())) {
            def validJobs = jobsWithUniqueValidJobId(parentJobs)
            def validIds = uniqueValidJobIDs(parentJobIDs).collect{ it.toString() }
            def idsOfValidJobs = jobs2jobIDs(validJobs).collect { it.toString() }
            if (validIds != idsOfValidJobs) {
                throw new RuntimeException("parentJobBEJob needs to be called with one of parentJobs, parentJobIDs, or parentJobsIDs and *corresponding* parentJobs.")
            }
            pJobs = validJobs
        } else if (null == parentJobIDs && null == parentJobs) {
            pJobs = new LinkedList<BEJob>()
        } else if (null != parentJobs) {
            pJobs = jobsWithUniqueValidJobId(parentJobs)
        } else {
            pJobs = uniqueValidJobIDs(parentJobIDs).collect { new BEJob(it, jobManager) }
        }
        return pJobs
    }

    Job(ExecutionContext context, String jobName, String toolID, String inlineScript, List<String> arrayIndices, Map<String, Object> inputParameters,
        List<BaseFile> parentFiles, List<BaseFile> filesToVerify) {
        super(new BEJobID()
                , jobName
                , context.getConfiguration().getProcessingToolPath(context, TOOLID_WRAPIN_SCRIPT)
                , inlineScript
                , inlineScript ? null : getToolMD5(TOOLID_WRAPIN_SCRIPT, context)
                , getResourceSetFromConfiguration(toolID, context)
                , []
                , [:]
                , Roddy.getJobManager()
                , JobLog.toOneFile(context.loggingDirectory)
                , null)
        this.localToolPath = context.getConfiguration().getSourceToolPath(toolID)
        this.addParentJobs(reconcileParentJobInformation(collectParentJobsFromFiles(parentFiles), collectDependencyIDsFromFiles(parentFiles), jobManager))
        this.context = context
        this.toolID = toolID<commit_msg>Fix the name of the job log to the $jobName.o$jobId.<commit_after>/*
 * Copyright (c) 2017 eilslabs.
 *
 * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt).
 */

package de.dkfz.roddy.execution.jobs

import de.dkfz.roddy.AvailableFeatureToggles
import de.dkfz.roddy.Constants
import de.dkfz.roddy.Roddy
import de.dkfz.roddy.StringConstants
import de.dkfz.roddy.config.*
import de.dkfz.roddy.config.converters.BashConverter
import de.dkfz.roddy.core.ExecutionContext
import de.dkfz.roddy.core.ExecutionContextError
import de.dkfz.roddy.core.ExecutionContextLevel
import de.dkfz.roddy.execution.io.fs.FileSystemAccessProvider
import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager
import de.dkfz.roddy.execution.jobs.direct.synchronousexecution.DirectSynchronousExecutionJobManager
import de.dkfz.roddy.knowledge.files.BaseFile
import de.dkfz.roddy.knowledge.files.FileGroup
import de.dkfz.roddy.tools.LoggerWrapper
import de.dkfz.roddy.tools.RoddyIOHelperMethods
import groovy.transform.CompileDynamic

import static de.dkfz.roddy.Constants.NO_VALUE
import static de.dkfz.roddy.config.FilenamePattern.PLACEHOLDER_JOBPARAMETER

@groovy.transform.CompileStatic
class Job extends BEJob<BEJob, BEJobResult> {

    private static final de.dkfz.roddy.tools.LoggerWrapper logger = de.dkfz.roddy.tools.LoggerWrapper.getLogger(BEJob.class.getSimpleName())

    public static final String TOOLID_WRAPIN_SCRIPT = "wrapinScript"
    public static final String PARM_WRAPPED_SCRIPT = "WRAPPED_SCRIPT"
    public static final String PARM_WRAPPED_SCRIPT_MD5 = "WRAPPED_SCRIPT_MD5"

    /**
     * The current context.
     */
    public final ExecutionContext context

    /**
     * The local tool path
     */
    private final File localToolPath

    /**
     * The tool you want to call.
     */
    private final String toolID

    /**
     * Keeps a list of all unchanged, initial parameters, including default job parameters.
     */
    protected final Map<String, Object> allRawInputParameters

    /**
     * Provide a list of files if you want to generate job dependencies.
     */
    public transient final List<BaseFile> parentFiles

    private Map<String, Object> initialInputParameters = [:]

    /**
     * Command object of last execution.
     */
    protected transient Command lastCommand

    /**
     * The list contains files which should be validated for a job restart.
     */
    public final List<BaseFile> filesToVerify

    Job(ExecutionContext context, String jobName, String toolID, List<String> arrayIndices, Map<String, Object> parameters, List<BaseFile> parentFiles) {
        this(context, jobName, toolID, arrayIndices, parameters, parentFiles, null)
    }

    Job(ExecutionContext context, String jobName, String toolID, Map<String, Object> parameters, List<BaseFile> parentFiles, List<BaseFile> filesToVerify) {
        this(context, jobName, toolID, null, parameters, parentFiles, filesToVerify)
    }

    Job(ExecutionContext context, String jobName, String toolID, Map<String, Object> parameters, List<BaseFile> parentFiles) {
        this(context, jobName, toolID, null, parameters, parentFiles, null)
    }

    Job(ExecutionContext context, String jobName, String toolID, List<String> arrayIndices, List<BaseFile> filesToVerify, Map<String, Object> parameters) {
        this(context, jobName, toolID, arrayIndices, parameters, null, filesToVerify)
    }

    Job(ExecutionContext context, String jobName, List<BaseFile> filesToVerify, String toolID, Map<String, Object> parameters) {
        this(context, jobName, toolID, null, parameters, null, filesToVerify)
    }

    Job(ExecutionContext context, String jobName, String toolID, List<String> arrayIndices, Map<String, Object> parameters) {
        this(context, jobName, toolID, arrayIndices, parameters, null, null)
    }

    Job(ExecutionContext context, String jobName, String toolID, Map<String, Object> parameters) {
        this(context, jobName, toolID, null, parameters, null, null)
    }

//    /**
//     * This is for job implementations which do the writeConfigurationFile on their own.
//     */
//    protected BEJob(String jobName, ExecutionContext context, String toolID, Map<String, String> parameters, List<String> arrayIndices, List<BaseFile> parentFiles, List<BaseFile> filesToVerify) {
//        super(jobName
//                , context.getConfiguration().getProcessingToolPath(context, toolID)
//                , toolID != null && toolID.trim().length() > 0 ? context.getConfiguration().getProcessingToolMD5(toolID): null
//                , arrayIndices
//                , parameters
//                , collectParentJobsFromFiles(parentFiles)
//                , collectDependencyIDsFromFiles(parentFiles)
//                , Roddy.getJobManager())
//        this.toolID = toolID
//        this.context = context
//        this.parentFiles = parentFiles
//        this.filesToVerify = filesToVerify
//        this.context.addExecutedJob(this)
//    }

    Job(ExecutionContext context, String jobName, String toolID, List<String> arrayIndices, Map<String, Object> inputParameters, List<BaseFile> parentFiles, List<BaseFile> filesToVerify) {
        this(context, jobName, toolID, null, arrayIndices, inputParameters, parentFiles, filesToVerify)
    }

    private static List<BEJobID> jobs2jobIDs(List<BEJob> jobs) {
        if (null == jobs) {
            return new LinkedList<BEJobID>()
        } else {
            return jobs.collect { it.runResult.jobID }
        }
    }

    private static List<BEJob> reconcileParentJobInformation(List<BEJob> parentJobs, List<BEJobID> parentJobIDs, BatchEuphoriaJobManager jobManager) {
        List<BEJob> pJobs
        if ((null != parentJobIDs && !parentJobIDs.isEmpty()) &&
                (null != parentJobs && !parentJobs.isEmpty())) {
            def validJobs = jobsWithUniqueValidJobId(parentJobs)
            def validIds = uniqueValidJobIDs(parentJobIDs).collect{ it.toString() }
            def idsOfValidJobs = jobs2jobIDs(validJobs).collect { it.toString() }
            if (validIds != idsOfValidJobs) {
                throw new RuntimeException("parentJobBEJob needs to be called with one of parentJobs, parentJobIDs, or parentJobsIDs and *corresponding* parentJobs.")
            }
            pJobs = validJobs
        } else if (null == parentJobIDs && null == parentJobs) {
            pJobs = new LinkedList<BEJob>()
        } else if (null != parentJobs) {
            pJobs = jobsWithUniqueValidJobId(parentJobs)
        } else {
            pJobs = uniqueValidJobIDs(parentJobIDs).collect { new BEJob(it, jobManager) }
        }
        return pJobs
    }

    Job(ExecutionContext context, String jobName, String toolID, String inlineScript, List<String> arrayIndices, Map<String, Object> inputParameters,
        List<BaseFile> parentFiles, List<BaseFile> filesToVerify) {
        super(new BEJobID()
                , jobName
                , context.getConfiguration().getProcessingToolPath(context, TOOLID_WRAPIN_SCRIPT)
                , inlineScript
                , inlineScript ? null : getToolMD5(TOOLID_WRAPIN_SCRIPT, context)
                , getResourceSetFromConfiguration(toolID, context)
                , []
                , [:]
                , Roddy.getJobManager()
                , JobLog.toOneFile(new File (context.loggingDirectory, jobName + ".o{JOB_ID}"))
                , null)
        this.localToolPath = context.getConfiguration().getSourceToolPath(toolID)
        this.addParentJobs(reconcileParentJobInformation(collectParentJobsFromFiles(parentFiles), collectDependencyIDsFromFiles(parentFiles), jobManager))
        this.context = context
        this.toolID = toolID
{'fraction_non_alphanumeric': 0.09464887276496502, 'fraction_numerical': 0.0014252396994039907, 'mean_word_length': 3.2690818584070795, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 87, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '131445', 'n_tokens_mistral': 4524, 'n_tokens_neox': 4433, 'n_words': 1030}