<commit_before>/*
 * TasselPipeline.java
 *
 * Created on June 11, 2009
 *
 */
package net.maizegenetics.pipeline;

import java.awt.Frame;

import java.io.BufferedReader;

import java.lang.reflect.Constructor;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.regex.Pattern;

import net.maizegenetics.baseplugins.AbstractDisplayPlugin;
import net.maizegenetics.baseplugins.CombineDataSetsPlugin;
import net.maizegenetics.baseplugins.ConvertAlignmentCoordinatesPlugin;
import net.maizegenetics.baseplugins.ConvertSBitTBitPlugin;
import net.maizegenetics.baseplugins.CreateTreePlugin;
import net.maizegenetics.baseplugins.DistanceMatrixPlugin;
import net.maizegenetics.baseplugins.DistanceMatrixRangesPlugin;
import net.maizegenetics.baseplugins.ExportMultiplePlugin;
import net.maizegenetics.baseplugins.FileLoadPlugin;
import net.maizegenetics.baseplugins.FilterAlignmentPlugin;
import net.maizegenetics.baseplugins.FilterSiteNamePlugin;
import net.maizegenetics.baseplugins.FilterTaxaAlignmentPlugin;
import net.maizegenetics.baseplugins.FilterTraitsPlugin;
import net.maizegenetics.baseplugins.FixedEffectLMPlugin;
import net.maizegenetics.baseplugins.FlapjackLoadPlugin;
import net.maizegenetics.baseplugins.GenotypeImputationPlugin;
import net.maizegenetics.baseplugins.GenotypeSummaryPlugin;
import net.maizegenetics.baseplugins.IntersectionAlignmentPlugin;
import net.maizegenetics.baseplugins.KinshipPlugin;
import net.maizegenetics.baseplugins.LinkageDiseqDisplayPlugin;
import net.maizegenetics.baseplugins.LinkageDisequilibriumPlugin;
import net.maizegenetics.baseplugins.MLMPlugin;
import net.maizegenetics.baseplugins.MergeAlignmentsPlugin;
import net.maizegenetics.baseplugins.MergeAlignmentsSameSitesPlugin;
import net.maizegenetics.baseplugins.NumericalGenotypePlugin;
import net.maizegenetics.baseplugins.PlinkLoadPlugin;
import net.maizegenetics.baseplugins.SeparatePlugin;
import net.maizegenetics.baseplugins.SequenceDiversityPlugin;
import net.maizegenetics.baseplugins.SynonymizerPlugin;
import net.maizegenetics.baseplugins.TableDisplayPlugin;
import net.maizegenetics.baseplugins.UnionAlignmentPlugin;
import net.maizegenetics.baseplugins.genomicselection.RidgeRegressionEmmaPlugin;
import net.maizegenetics.gbs.maps.TagsOnPhysMapHDF5;
import net.maizegenetics.gbs.maps.TagsOnPhysicalMap;

import net.maizegenetics.pal.alignment.Alignment;
import net.maizegenetics.pal.gui.LinkageDisequilibriumComponent;
import net.maizegenetics.pal.popgen.LinkageDisequilibrium.testDesign;

import net.maizegenetics.pal.ids.Identifier;
import net.maizegenetics.pal.ids.SimpleIdGroup;
import net.maizegenetics.pal.popgen.LinkageDisequilibrium.HetTreatment;

import net.maizegenetics.plugindef.AbstractPlugin;
import net.maizegenetics.plugindef.DataSet;
import net.maizegenetics.plugindef.Datum;
import net.maizegenetics.plugindef.Plugin;
import net.maizegenetics.plugindef.PluginEvent;
import net.maizegenetics.plugindef.PluginListener;
import net.maizegenetics.plugindef.ThreadedPluginListener;

import net.maizegenetics.prefs.TasselPrefs;
import net.maizegenetics.progress.ProgressPanel;
import net.maizegenetics.tassel.DataTreePanel;
import net.maizegenetics.tassel.TASSELMainFrame;
import net.maizegenetics.util.ExceptionUtils;
import net.maizegenetics.util.Utils;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

/**
 *
 * @author terryc
 */
public class TasselPipeline implements PluginListener {

    private static final Logger myLogger = Logger.getLogger(TasselPipeline.class);
    private final TASSELMainFrame myMainFrame;
    private final Map<String, List> myForks = new LinkedHashMap<String, List>();
    private String myCurrentFork = null;
    private List<Plugin> myCurrentPipe = null;
    private Plugin myFirstPlugin = null;
    private final List<ThreadedPluginListener> myThreads = new ArrayList();
    private final Map<Plugin, Integer> myProgressValues = new HashMap<Plugin, Integer>();

    /**
     * Creates a new instance of TasselPipeline
     */
    public TasselPipeline(String args[], TASSELMainFrame frame) {

        myMainFrame = frame;

        java.util.Properties props = new java.util.Properties();
        props.setProperty("log4j.logger.net.maizegenetics", "INFO, stdout");
        props.setProperty("log4j.appender.stdout",
                "org.apache.log4j.ConsoleAppender");
        props.setProperty("log4j.appender.stdout.layout",
                "org.apache.log4j.TTCCLayout");
        PropertyConfigurator.configure(props);

        try {

            if ((args.length == 1) && (args[0].equalsIgnoreCase("-versionComment"))) {
                System.out.println("Version " + myMainFrame.version + " on " + myMainFrame.versionDate);
                return;
            }

            if ((args.length == 1) && (args[0].equalsIgnoreCase("-versionTag"))) {
                System.out.println("V" + myMainFrame.version);
                return;
            }

            myLogger.info("Tassel Version: " + myMainFrame.version + "  Date: " + myMainFrame.versionDate);
            myLogger.info("Max Available Memory Reported by JVM: " + Utils.getMaxHeapSizeMB() + " MB");

            parseArgs(args);

            if (myMainFrame != null) {
                ProgressPanel progressPanel = myMainFrame.getProgressPanel();
                if (progressPanel != null) {
                    Iterator itr = myForks.keySet().iterator();
                    while (itr.hasNext()) {
                        String key = (String) itr.next();
                        List<Plugin> current = myForks.get(key);
                        progressPanel.addPipelineSegment(current);
                    }
                }
            }

            for (int i = 0; i < myThreads.size(); i++) {
                ThreadedPluginListener current = (ThreadedPluginListener) myThreads.get(i);
                current.start();
            }

        } catch (Exception e) {
            System.exit(1);
        }

    }

    public static void main(String args[]) {
        if ((args.length >= 2) && (args[0].equalsIgnoreCase("-createXML"))) {
            String xmlFilename = args[1].trim();
            String[] temp = new String[args.length - 2];
            System.arraycopy(args, 2, temp, 0, temp.length);
            TasselPipelineXMLUtil.writeArgsAsXML(xmlFilename, temp);
        } else if ((args.length >= 2) && (args[0].equalsIgnoreCase("-translateXML"))) {
            String xmlFilename = args[1].trim();
            String[] result = TasselPipelineXMLUtil.readXMLAsArgs(xmlFilename);
            for (int i = 0; i < result.length; i++) {
                System.out.print(result[i]);
                System.out.print(" ");
            }
            System.out.println("");
        } else {
            TasselPrefs.setPersistPreferences(false);
            new TasselPipeline(args, null);
        }
    }

    public void parseArgs(String[] args) {

        if ((args.length >= 2) && (args[0].equalsIgnoreCase("-configFile"))) {
            String xmlFilename = args[1].trim();
            args = TasselPipelineXMLUtil.readXMLAsArgs(xmlFilename);
        }

        int index = 0;
        while (index < args.length) {

            try {

                String current = args[index++];

                if (!current.startsWith("-")) {
                    throw new IllegalArgumentException("TasselPipeline: parseArgs: expecting argument beginning with dash: " + current);
                }<commit_msg>Replace first em-dash with en-dash in Tassel Pipeline Arguments to accept em-dash and en-dash.<commit_after>/*
 * TasselPipeline.java
 *
 * Created on June 11, 2009
 *
 */
package net.maizegenetics.pipeline;

import java.awt.Frame;

import java.io.BufferedReader;

import java.lang.reflect.Constructor;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.regex.Pattern;

import net.maizegenetics.baseplugins.AbstractDisplayPlugin;
import net.maizegenetics.baseplugins.CombineDataSetsPlugin;
import net.maizegenetics.baseplugins.ConvertAlignmentCoordinatesPlugin;
import net.maizegenetics.baseplugins.ConvertSBitTBitPlugin;
import net.maizegenetics.baseplugins.CreateTreePlugin;
import net.maizegenetics.baseplugins.DistanceMatrixPlugin;
import net.maizegenetics.baseplugins.DistanceMatrixRangesPlugin;
import net.maizegenetics.baseplugins.ExportMultiplePlugin;
import net.maizegenetics.baseplugins.FileLoadPlugin;
import net.maizegenetics.baseplugins.FilterAlignmentPlugin;
import net.maizegenetics.baseplugins.FilterSiteNamePlugin;
import net.maizegenetics.baseplugins.FilterTaxaAlignmentPlugin;
import net.maizegenetics.baseplugins.FilterTraitsPlugin;
import net.maizegenetics.baseplugins.FixedEffectLMPlugin;
import net.maizegenetics.baseplugins.FlapjackLoadPlugin;
import net.maizegenetics.baseplugins.GenotypeImputationPlugin;
import net.maizegenetics.baseplugins.GenotypeSummaryPlugin;
import net.maizegenetics.baseplugins.IntersectionAlignmentPlugin;
import net.maizegenetics.baseplugins.KinshipPlugin;
import net.maizegenetics.baseplugins.LinkageDiseqDisplayPlugin;
import net.maizegenetics.baseplugins.LinkageDisequilibriumPlugin;
import net.maizegenetics.baseplugins.MLMPlugin;
import net.maizegenetics.baseplugins.MergeAlignmentsPlugin;
import net.maizegenetics.baseplugins.MergeAlignmentsSameSitesPlugin;
import net.maizegenetics.baseplugins.NumericalGenotypePlugin;
import net.maizegenetics.baseplugins.PlinkLoadPlugin;
import net.maizegenetics.baseplugins.SeparatePlugin;
import net.maizegenetics.baseplugins.SequenceDiversityPlugin;
import net.maizegenetics.baseplugins.SynonymizerPlugin;
import net.maizegenetics.baseplugins.TableDisplayPlugin;
import net.maizegenetics.baseplugins.UnionAlignmentPlugin;
import net.maizegenetics.baseplugins.genomicselection.RidgeRegressionEmmaPlugin;
import net.maizegenetics.gbs.maps.TagsOnPhysMapHDF5;
import net.maizegenetics.gbs.maps.TagsOnPhysicalMap;

import net.maizegenetics.pal.alignment.Alignment;
import net.maizegenetics.pal.gui.LinkageDisequilibriumComponent;
import net.maizegenetics.pal.popgen.LinkageDisequilibrium.testDesign;

import net.maizegenetics.pal.ids.Identifier;
import net.maizegenetics.pal.ids.SimpleIdGroup;
import net.maizegenetics.pal.popgen.LinkageDisequilibrium.HetTreatment;

import net.maizegenetics.plugindef.AbstractPlugin;
import net.maizegenetics.plugindef.DataSet;
import net.maizegenetics.plugindef.Datum;
import net.maizegenetics.plugindef.Plugin;
import net.maizegenetics.plugindef.PluginEvent;
import net.maizegenetics.plugindef.PluginListener;
import net.maizegenetics.plugindef.ThreadedPluginListener;

import net.maizegenetics.prefs.TasselPrefs;
import net.maizegenetics.progress.ProgressPanel;
import net.maizegenetics.tassel.DataTreePanel;
import net.maizegenetics.tassel.TASSELMainFrame;
import net.maizegenetics.util.ExceptionUtils;
import net.maizegenetics.util.Utils;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

/**
 *
 * @author terryc
 */
public class TasselPipeline implements PluginListener {

    private static final Logger myLogger = Logger.getLogger(TasselPipeline.class);
    private final TASSELMainFrame myMainFrame;
    private final Map<String, List> myForks = new LinkedHashMap<String, List>();
    private String myCurrentFork = null;
    private List<Plugin> myCurrentPipe = null;
    private Plugin myFirstPlugin = null;
    private final List<ThreadedPluginListener> myThreads = new ArrayList();
    private final Map<Plugin, Integer> myProgressValues = new HashMap<Plugin, Integer>();

    /**
     * Creates a new instance of TasselPipeline
     */
    public TasselPipeline(String args[], TASSELMainFrame frame) {

        myMainFrame = frame;

        java.util.Properties props = new java.util.Properties();
        props.setProperty("log4j.logger.net.maizegenetics", "INFO, stdout");
        props.setProperty("log4j.appender.stdout",
                "org.apache.log4j.ConsoleAppender");
        props.setProperty("log4j.appender.stdout.layout",
                "org.apache.log4j.TTCCLayout");
        PropertyConfigurator.configure(props);

        try {

            if ((args.length == 1) && (args[0].equalsIgnoreCase("-versionComment"))) {
                System.out.println("Version " + myMainFrame.version + " on " + myMainFrame.versionDate);
                return;
            }

            if ((args.length == 1) && (args[0].equalsIgnoreCase("-versionTag"))) {
                System.out.println("V" + myMainFrame.version);
                return;
            }

            myLogger.info("Tassel Version: " + myMainFrame.version + "  Date: " + myMainFrame.versionDate);
            myLogger.info("Max Available Memory Reported by JVM: " + Utils.getMaxHeapSizeMB() + " MB");

            parseArgs(args);

            if (myMainFrame != null) {
                ProgressPanel progressPanel = myMainFrame.getProgressPanel();
                if (progressPanel != null) {
                    Iterator itr = myForks.keySet().iterator();
                    while (itr.hasNext()) {
                        String key = (String) itr.next();
                        List<Plugin> current = myForks.get(key);
                        progressPanel.addPipelineSegment(current);
                    }
                }
            }

            for (int i = 0; i < myThreads.size(); i++) {
                ThreadedPluginListener current = (ThreadedPluginListener) myThreads.get(i);
                current.start();
            }

        } catch (Exception e) {
            System.exit(1);
        }

    }

    public static void main(String args[]) {
        if ((args.length >= 2) && (args[0].equalsIgnoreCase("-createXML"))) {
            String xmlFilename = args[1].trim();
            String[] temp = new String[args.length - 2];
            System.arraycopy(args, 2, temp, 0, temp.length);
            TasselPipelineXMLUtil.writeArgsAsXML(xmlFilename, temp);
        } else if ((args.length >= 2) && (args[0].equalsIgnoreCase("-translateXML"))) {
            String xmlFilename = args[1].trim();
            String[] result = TasselPipelineXMLUtil.readXMLAsArgs(xmlFilename);
            for (int i = 0; i < result.length; i++) {
                System.out.print(result[i]);
                System.out.print(" ");
            }
            System.out.println("");
        } else {
            TasselPrefs.setPersistPreferences(false);
            new TasselPipeline(args, null);
        }
    }

    public void parseArgs(String[] args) {

        if ((args.length >= 2) && (args[0].equalsIgnoreCase("-configFile"))) {
            String xmlFilename = args[1].trim();
            args = TasselPipelineXMLUtil.readXMLAsArgs(xmlFilename);
        }

        int index = 0;
        while (index < args.length) {

            try {

                String current = args[index++];
                current = current.replaceFirst("—", "-");

                if (!current.startsWith("-")) {
                    throw new IllegalArgumentException("TasselPipeline: parseArgs: expecting argument beginning with dash: " + current);
                }
{'fraction_non_alphanumeric': 0.10390700678075557, 'fraction_numerical': 0.004520503713270907, 'mean_word_length': 3.3956854953164917, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '524120', 'n_tokens_mistral': 4572, 'n_tokens_neox': 4451, 'n_words': 765}