<commit_before>/*
 * Copyright (c) 2004-2022 The MZmine Development Team
 *
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
 * files (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use,
 * copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following
 * conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

package io.github.mzmine.modules.batchmode;

import io.github.mzmine.datamodel.MZmineProject;
import io.github.mzmine.datamodel.RawDataFile;
import io.github.mzmine.datamodel.features.FeatureList;
import io.github.mzmine.main.GoogleAnalyticsTracker;
import io.github.mzmine.main.MZmineCore;
import io.github.mzmine.modules.MZmineProcessingModule;
import io.github.mzmine.modules.MZmineProcessingStep;
import io.github.mzmine.modules.io.import_rawdata_all.AllSpectralDataImportParameters;
import io.github.mzmine.parameters.Parameter;
import io.github.mzmine.parameters.ParameterSet;
import io.github.mzmine.parameters.parametertypes.EmbeddedParameterSet;
import io.github.mzmine.parameters.parametertypes.filenames.FileNameParameter;
import io.github.mzmine.parameters.parametertypes.filenames.FileNamesParameter;
import io.github.mzmine.parameters.parametertypes.selectors.FeatureListsParameter;
import io.github.mzmine.parameters.parametertypes.selectors.FeatureListsSelection;
import io.github.mzmine.parameters.parametertypes.selectors.RawDataFilesParameter;
import io.github.mzmine.parameters.parametertypes.selectors.RawDataFilesSelection;
import io.github.mzmine.taskcontrol.AbstractTask;
import io.github.mzmine.taskcontrol.Task;
import io.github.mzmine.taskcontrol.TaskPriority;
import io.github.mzmine.taskcontrol.TaskStatus;
import io.github.mzmine.taskcontrol.impl.WrappedTask;
import io.github.mzmine.util.ExitCode;
import io.github.mzmine.util.files.FileAndPathUtil;
import java.io.File;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.logging.Logger;
import org.jetbrains.annotations.NotNull;

/**
 * Batch mode task
 */
public class BatchTask extends AbstractTask {

  private final BatchQueue queue;
  private final ParameterSet parameters;
  private final List<File> subDirectories;
  private final Logger logger = Logger.getLogger(this.getClass().getName());
  private final int totalSteps;
  private final MZmineProject project;
  private final int datasets;
  private int processedSteps;
  private List<RawDataFile> createdDataFiles, previousCreatedDataFiles, startDataFiles;
  private List<FeatureList> createdFeatureLists, previousCreatedFeatureLists, startFeatureLists;
  private final int stepsPerDataset;

  BatchTask(MZmineProject project, ParameterSet parameters, @NotNull Instant moduleCallDate) {
    this(project, parameters, moduleCallDate,
        null); // we don't create any new data here, date is irrelevant, too.
  }

  public BatchTask(final MZmineProject project, final ParameterSet parameters,
      final Instant moduleCallDate, final List<File> subDirectories) {
    super(null, moduleCallDate);
    this.project = project;
    this.queue = parameters.getParameter(BatchModeParameters.batchQueue).getValue();
    this.parameters = parameters;
    // if sub directories is set - the input and output files are changed to each sub directory
    // each sub dir is processed sequentially as a different dataset
    boolean value = parameters.getParameter(BatchModeParameters.advanced).getValue();
    this.subDirectories = value ? subDirectories : null;
    datasets = subDirectories == null || subDirectories.isEmpty() ? 1 : subDirectories.size();
    stepsPerDataset = queue.size();
    totalSteps = stepsPerDataset * datasets;
    createdDataFiles = new ArrayList<>();
    createdFeatureLists = new ArrayList<>();
    previousCreatedDataFiles = new ArrayList<>();
    previousCreatedFeatureLists = new ArrayList<>();
  }

  @Override
  public void run() {

    setStatus(TaskStatus.PROCESSING);
    logger.info("Starting a batch of " + totalSteps + " steps");

    // unmodifiable copies of currents lists
    startFeatureLists = project.getCurrentFeatureLists();
    startDataFiles = project.getCurrentRawDataFiles();

    // advanced parameters
    AdvancedBatchModeParameters advanced = parameters.getParameter(BatchModeParameters.advanced)
        .getEmbeddedParameters();
    boolean skipOnError = advanced.getValue(AdvancedBatchModeParameters.skipOnError);
    boolean searchSubdirs = advanced.getValue(AdvancedBatchModeParameters.includeSubdirectories);
    boolean createResultsDir = advanced.getValue(
        AdvancedBatchModeParameters.createResultsDirectory);
    File parentDir = advanced.getValue(AdvancedBatchModeParameters.processingParentDir);

    int errorDataset = 0;
    int currentDataset = -1;
    String datasetName = "";
    // Process individual batch steps
    for (int i = 0; i < totalSteps; i++) {
      // at the end of one dataset, clear the project and start over again
      if (processedSteps % stepsPerDataset == 0) {
        // clear the old project
        MZmineCore.getProjectManager().clearProject();
        currentDataset++;
        logger.info(String.format("Starting batch dataset %d/%d", processedSteps / stepsPerDataset,
            datasets));

        // change files
        File datasetDir = subDirectories.get(currentDataset);
        datasetName = datasetDir.getName();
        File[] allFiles = FileAndPathUtil.findFilesInDirFlat(datasetDir,
            AllSpectralDataImportParameters.ALL_MS_DATA_FILTER, searchSubdirs);

        if (allFiles.length != 0) {
          // set files to import
          setImportFiles(allFiles);

          // set files to output
          setOutputFiles(parentDir, createResultsDir, datasetName);
        } else {
          errorDataset++;
          logger.info("No data files found in directory: " + datasetName);
          // no data files
          if (skipOnError) {
            processedSteps += stepsPerDataset;
            continue;
          } else {
            setStatus(TaskStatus.ERROR);
            setErrorMessage("No data files found in directory: " + datasetName);
            return;
          }
        }
      }

      // run step
      processQueueStep(i % stepsPerDataset);
      processedSteps++;

      // If we are canceled or ran into error, stop here
      if (isCanceled()) {
        return;
      } else if (getStatus() == TaskStatus.ERROR) {
        errorDataset++;
        if (skipOnError && datasets - currentDataset > 0) {
          // skip to next dataset
          logger.info("Error in dataset: " + datasetName + " total error datasets:" + errorDataset);
          processedSteps = (processedSteps / stepsPerDataset + 1) * stepsPerDataset;
          continue;
        } else {
          return;
        }
      }
    }

    logger.info("Finished a batch of " + totalSteps + " steps");
    setStatus(TaskStatus.FINISHED);
  }

  private void setOutputFiles(final File parentDir, final boolean createResultsDir,
      final String datasetName) {
    int changedOutputSteps = 0;
    for (MZmineProcessingStep<?> currentStep : queue) {
      ParameterSet params = currentStep.getParameterSet();
      // check some
      for (final Parameter<?> p : params.getParameters()) {
        if (p instanceof FileNameParameter fnp) {
          File old = fnp.getValue();
          String oldName = FileAndPathUtil.eraseFormat(old).getName();
          fnp.setValue(createDatasetExportPath(parentDir, createResultsDir, datasetName, oldName));
          changedOutputSteps++;
        }
      }
    }
    if (changedOutputSteps == 0) {
      logger.info(
          "No output steps were changes... Make sure to include steps that include filename parameters for export");
      throw new IllegalStateException(
          "No output steps were changes... Make sure to include steps that include filename parameters for export");
    } else {
      logger.info("Changed output for n steps=" + changedOutputSteps);
    }
  }

  private File createDatasetExportPath(final File parentDir, final boolean createResultsDir,
      final String datasetName, final String oldName) {
    if (createResultsDir) {
      return Paths.get(parentDir.getPath(), "results", datasetName, oldName + datasetName).toFile();
    } else {
      return Paths.get(parentDir.getPath(), datasetName, oldName + datasetName).toFile();
    }
  }

  private void setImportFiles(final File[] allFiles) {<commit_msg>Revisions, only change export modules<commit_after>/*
 * Copyright (c) 2004-2022 The MZmine Development Team
 *
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
 * files (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use,
 * copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following
 * conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

package io.github.mzmine.modules.batchmode;

import io.github.mzmine.datamodel.MZmineProject;
import io.github.mzmine.datamodel.RawDataFile;
import io.github.mzmine.datamodel.features.FeatureList;
import io.github.mzmine.main.GoogleAnalyticsTracker;
import io.github.mzmine.main.MZmineCore;
import io.github.mzmine.modules.MZmineModuleCategory;
import io.github.mzmine.modules.MZmineProcessingModule;
import io.github.mzmine.modules.MZmineProcessingStep;
import io.github.mzmine.modules.MZmineRunnableModule;
import io.github.mzmine.modules.io.import_rawdata_all.AllSpectralDataImportParameters;
import io.github.mzmine.parameters.Parameter;
import io.github.mzmine.parameters.ParameterSet;
import io.github.mzmine.parameters.parametertypes.EmbeddedParameterSet;
import io.github.mzmine.parameters.parametertypes.filenames.FileNameParameter;
import io.github.mzmine.parameters.parametertypes.filenames.FileNamesParameter;
import io.github.mzmine.parameters.parametertypes.selectors.FeatureListsParameter;
import io.github.mzmine.parameters.parametertypes.selectors.FeatureListsSelection;
import io.github.mzmine.parameters.parametertypes.selectors.RawDataFilesParameter;
import io.github.mzmine.parameters.parametertypes.selectors.RawDataFilesSelection;
import io.github.mzmine.taskcontrol.AbstractTask;
import io.github.mzmine.taskcontrol.Task;
import io.github.mzmine.taskcontrol.TaskPriority;
import io.github.mzmine.taskcontrol.TaskStatus;
import io.github.mzmine.taskcontrol.impl.WrappedTask;
import io.github.mzmine.util.ExitCode;
import io.github.mzmine.util.files.FileAndPathUtil;
import java.io.File;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.logging.Logger;
import org.jetbrains.annotations.NotNull;

/**
 * Batch mode task
 */
public class BatchTask extends AbstractTask {

  private final BatchQueue queue;
  private final ParameterSet parameters;
  private final List<File> subDirectories;
  private final Logger logger = Logger.getLogger(this.getClass().getName());
  private final int totalSteps;
  private final MZmineProject project;
  private final int datasets;
  private int processedSteps;
  private List<RawDataFile> createdDataFiles, previousCreatedDataFiles, startDataFiles;
  private List<FeatureList> createdFeatureLists, previousCreatedFeatureLists, startFeatureLists;
  private final int stepsPerDataset;

  BatchTask(MZmineProject project, ParameterSet parameters, @NotNull Instant moduleCallDate) {
    this(project, parameters, moduleCallDate,
        null); // we don't create any new data here, date is irrelevant, too.
  }

  public BatchTask(final MZmineProject project, final ParameterSet parameters,
      final Instant moduleCallDate, final List<File> subDirectories) {
    super(null, moduleCallDate);
    this.project = project;
    this.queue = parameters.getParameter(BatchModeParameters.batchQueue).getValue();
    this.parameters = parameters;
    // if sub directories is set - the input and output files are changed to each sub directory
    // each sub dir is processed sequentially as a different dataset
    boolean value = parameters.getParameter(BatchModeParameters.advanced).getValue();
    this.subDirectories = value ? subDirectories : null;
    datasets = subDirectories == null || subDirectories.isEmpty() ? 1 : subDirectories.size();
    stepsPerDataset = queue.size();
    totalSteps = stepsPerDataset * datasets;
    createdDataFiles = new ArrayList<>();
    createdFeatureLists = new ArrayList<>();
    previousCreatedDataFiles = new ArrayList<>();
    previousCreatedFeatureLists = new ArrayList<>();
  }

  @Override
  public void run() {

    setStatus(TaskStatus.PROCESSING);
    logger.info("Starting a batch of " + totalSteps + " steps");

    // unmodifiable copies of currents lists
    startFeatureLists = project.getCurrentFeatureLists();
    startDataFiles = project.getCurrentRawDataFiles();

    // advanced parameters
    AdvancedBatchModeParameters advanced = parameters.getParameter(BatchModeParameters.advanced)
        .getEmbeddedParameters();
    boolean skipOnError = advanced.getValue(AdvancedBatchModeParameters.skipOnError);
    boolean searchSubdirs = advanced.getValue(AdvancedBatchModeParameters.includeSubdirectories);
    boolean createResultsDir = advanced.getValue(
        AdvancedBatchModeParameters.createResultsDirectory);
    File parentDir = advanced.getValue(AdvancedBatchModeParameters.processingParentDir);

    int errorDataset = 0;
    int currentDataset = -1;
    String datasetName = "";
    // Process individual batch steps
    for (int i = 0; i < totalSteps; i++) {
      // at the end of one dataset, clear the project and start over again
      if (processedSteps % stepsPerDataset == 0) {
        // clear the old project
        MZmineCore.getProjectManager().clearProject();
        currentDataset++;

        // change files
        File datasetDir = subDirectories.get(currentDataset);
        datasetName = datasetDir.getName();
        File[] allFiles = FileAndPathUtil.findFilesInDirFlat(datasetDir,
            AllSpectralDataImportParameters.ALL_MS_DATA_FILTER, searchSubdirs);

        logger.info(
            String.format("Processing batch dataset %s (%d/%d)", datasetName, currentDataset + 1,
                datasets));

        if (allFiles.length != 0) {
          // set files to import
          setImportFiles(allFiles);

          // set files to output
          setOutputFiles(parentDir, createResultsDir, datasetName);
        } else {
          errorDataset++;
          logger.info("No data files found in directory: " + datasetName);
          // no data files
          if (skipOnError) {
            processedSteps += stepsPerDataset;
            continue;
          } else {
            setStatus(TaskStatus.ERROR);
            setErrorMessage("No data files found in directory: " + datasetName);
            return;
          }
        }
      }

      // run step
      processQueueStep(i % stepsPerDataset);
      processedSteps++;

      // If we are canceled or ran into error, stop here
      if (isCanceled()) {
        return;
      } else if (getStatus() == TaskStatus.ERROR) {
        errorDataset++;
        if (skipOnError && datasets - currentDataset > 0) {
          // skip to next dataset
          logger.info("Error in dataset: " + datasetName + " total error datasets:" + errorDataset);
          processedSteps = (processedSteps / stepsPerDataset + 1) * stepsPerDataset;
          continue;
        } else {
          return;
        }
      }
    }

    logger.info("Finished a batch of " + totalSteps + " steps");
    setStatus(TaskStatus.FINISHED);
  }

  private void setOutputFiles(final File parentDir, final boolean createResultsDir,
      final String datasetName) {
    int changedOutputSteps = 0;
    for (MZmineProcessingStep<?> currentStep : queue) {
      // only change for export modules
      if (currentStep.getModule() instanceof MZmineRunnableModule mod && (
          mod.getModuleCategory() == MZmineModuleCategory.FEATURELISTEXPORT
              || mod.getModuleCategory() == MZmineModuleCategory.RAWDATAEXPORT)) {
        ParameterSet params = currentStep.getParameterSet();
        for (final Parameter<?> p : params.getParameters()) {
          if (p instanceof FileNameParameter fnp) {
            File old = fnp.getValue();
            String oldName = FileAndPathUtil.eraseFormat(old).getName();
            fnp.setValue(
                createDatasetExportPath(parentDir, createResultsDir, datasetName, oldName));
            changedOutputSteps++;
          }
        }
      }
    }
    if (changedOutputSteps == 0) {
      logger.info(
          "No output steps were changes... Make sure to include steps that include filename parameters for export");
      throw new IllegalStateException(
          "No output steps were changes... Make sure to include steps that include filename parameters for export");
    } else {
      logger.info("Changed output for n steps=" + changedOutputSteps);
    }
  }

  private File createDatasetExportPath(final File parentDir, final boolean createResultsDir,
      final String datasetName, final String oldName) {
    if (createResultsDir) {
      return Paths.get(parentDir.getPath(), "results", datasetName, oldName + datasetName).toFile();
    } else {
      return Paths.get(parentDir.getPath(), datasetName, oldName + datasetName).toFile();
    }
  }

  private void setImportFiles(final File[] allFiles) {
{'fraction_non_alphanumeric': 0.08720777515103756, 'fraction_numerical': 0.0019437877593905962, 'mean_word_length': 3.639532049719717, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '531744', 'n_tokens_mistral': 5097, 'n_tokens_neox': 4983, 'n_words': 1442}