<commit_before>/*******************************************************************************
 * In the Hi-WAY project we propose a novel approach of executing scientific
 * workflows processing Big Data, as found in NGS applications, on distributed
 * computational infrastructures. The Hi-WAY software stack comprises the func-
 * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster
 * for Apache Hadoop 2.x (YARN).
 *
 * List of Contributors:
 *
 * <NAME> (HU Berlin)
 * <NAME> (HU Berlin)
 * <NAME> (HU Berlin)
 * <NAME> (HU Berlin)
 *
 * <NAME> is funded by the European Commission through the BiobankCloud
 * project. <NAME> is funded by the Deutsche Forschungsgemeinschaft through
 * research training group SOAMED (GRK 1651).
 *
 * Copyright 2014 Humboldt-Universität zu Berlin
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package de.huberlin.wbi.hiway.am.galaxy;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr;
import de.huberlin.wbi.hiway.common.Data;
import de.huberlin.wbi.hiway.common.TaskInstance;

/**
 * An object that extends the Hi-WAY task instance object with its Galaxy tool and tool state, along with methods and structures to utilize and populate them.
 * 
 * @author <NAME>
 *
 */
public class GalaxyTaskInstance extends TaskInstance {
	// the Galaxy tool invoked by this task instance
	private GalaxyTool galaxyTool;
	// input data, which the task instance has to know about for determining the metadata using the Python classes provided by Galaxy
	private Set<GalaxyData> inputs;
	// the Python script run before the actual task instance execution that is responsible for populating the tool state with metadata
	private StringBuilder paramScript;
	// the post script that is to be run subsequent to the actual task invocation and that may involve the moving of files from temporary fodlers to their
	// destination
	private String postScript;
	// the tool state (i.e., the parameter settings in JSON format) of this task instance
	private JSONObject toolState;

	public GalaxyTaskInstance(long id, UUID workflowId, String taskName, GalaxyTool galaxyTool, String galaxyPath) {
		super(id, workflowId, taskName, Math.abs(taskName.hashCode() + 1), ForeignLambdaExpr.LANGID_BASH);
		this.galaxyTool = galaxyTool;
		toolState = new JSONObject();
		paramScript = new StringBuilder("import os, ast\nimport cPickle as pickle\nimport galaxy.app\n");
		this.postScript = "";
		inputs = new HashSet<>();

		// the task instance's invocScript variable is set here for it to be passed to the Worker thread, which writes the script's content as JsonReportEntry
		// to the log
		setInvocScript("script.sh");

		// As opposed to other Hi-WAY applciation masters, the Galaxy AM ha a fairly static command that can be build at task instance creation time
		StringBuilder commandSb = new StringBuilder();
		commandSb.append("PYTHONPATH=" + galaxyPath + "/lib:$PYTHONPATH; export PYTHONPATH\n");
		commandSb.append("PYTHON_EGG_CACHE=.; export PYTHON_EGG_CACHE\n");
		commandSb.append("python params.py\n");
		commandSb.append("cat pre.sh > script.sh\n");
		commandSb.append("echo `cheetah fill template.tmpl --pickle params.p -p` >> script.sh\n");
		commandSb.append("cat post.sh >> script.sh\n");
		commandSb.append("bash ").append(getInvocScript()).append("\n");
		setCommand(commandSb.toString());
	}

	/**
	 * A method that is called when a file name is encountered in the workflow description and the tool state has to be adjusted accordingly
	 * 
	 * @param name
	 *            the parameter name for this file, as specified in the workflow description
	 * @param computeMetadata
	 *            a parameter that determines whether metadata is to be computed for this file, which will only be the case for input data
	 * @param data
	 *            the Hi-WAY data object for this file
	 */
	public void addFile(String name, boolean computeMetadata, GalaxyData data) {
		addFile(name, computeMetadata, data, toolState);
	}

	/**
	 * A (recursive) method that is passed a file along with the JSON object in this task instance's tool state that corresponds to this file. This method is
	 * used to populate the tool state with information on the file.
	 * 
	 * @param name
	 *            the parameter name for this file, as specified in the workflow description
	 * @param computeMetadata
	 *            a parameter that determines whether metadata is to be computed for this file, which will only be the case for input data
	 * @param data
	 *            the Hi-WAY data object for this file
	 * @param jo
	 *            the JSON object in the tool state that corresponds to this file parameter
	 */
	private void addFile(String name, boolean computeMetadata, GalaxyData data, JSONObject jo) {
		try {
			Pattern p = Pattern.compile("(_[0-9]*)?\\|");
			Matcher m = p.matcher(name);

			// (1) if the to-be-added file is part of a repeat (and thus can be identified by an underscore and index number in its name), compute its prefix
			// (the repeat name) as well as its suffix (the actual parameter name) and index; use the prefix and index to obtain its JSON object from the tool
			// state and (recursively) call this method to proceed to (2)
			if (m.find()) {
				String prefix = name.substring(0, m.start());
				String suffix = name.substring(m.end());
				if (m.end() - m.start() > 2) {
					int index = Integer.parseInt(name.substring(m.start() + 1, m.end() - 1));
					JSONArray repeatJa = jo.getJSONArray(prefix);
					for (int i = 0; i < repeatJa.length(); i++) {
						JSONObject repeatJo = repeatJa.getJSONObject(i);
						if (repeatJo.getInt("__index__") == index) {
							addFile(suffix, computeMetadata, data, repeatJo);
							break;
						}
					}
				} else {
					addFile(suffix, computeMetadata, data, jo.getJSONObject(prefix));
				}
				// (2) fix both the template of this tool and the tool state of the task instance calling this method, such that they are in compliance with one
				// another when the tool state is used to set the parameters of the task instance at execution time
			} else {
				// (a) add several properties for this parameter to the tool state
				String fileName = data.getName();
				JSONObject fileJo = new JSONObject();
				fileJo.putOpt("path", fileName);
				fileJo.putOpt("name", fileName.split("\\.(?=[^\\.]+$)")[0]);
				fileJo.putOpt("files_path", data.getLocalDirectory());
				if (data.hasDataType()) {
					GalaxyDataType dataType = data.getDataType();
					String fileExt = dataType.getExtension();
					fileJo.putOpt("extension", fileExt);
					fileJo.putOpt("ext", fileExt);
				}
				// note that this metadata is an empty dictionary that will only be be filled for input data of the task instance calling this method; this is
				// done by executing a python script designed to populate the tool state with metadata prior to execution
				if (data.hasDataType())
					fileJo.putOpt("metadata", new JSONObject());
				jo.putOpt(name, fileJo);

				// (b) adjust the Python script for setting parameters at runtime to compute metadata for this file, given the computeMetadata parameter is set
				if (computeMetadata && data.hasDataType()) {
					GalaxyDataType dataType = data.getDataType();
					paramScript.append("from ");
					paramScript.append(dataType.getFile());
					paramScript.append(" import ");
					paramScript.append(dataType.getName());
					paramScript.append("\n");
					inputs.add(data);
				}
			}
		} catch (JSONException e) {
			e.printStackTrace();
			System.exit(-1);
		}
	}

	/**
	 * A method for setting the tool state of this task instance from the raw string of the workflow description file to the JSON object provided in the Python
	 * parameter script. There are several string operations that have to be performed before the tool state can be interpreted as a JSON object.
	 * 
	 * @param tool_state
	 *            the tool state string, as extracted from the workflow description file
	 * 
	 */
	public void addToolState(String tool_state) {
		String tool_state_json = tool_state;
		tool_state_json = tool_state_json.replaceAll("\"null\"", "\"\"");
		// replace "{ }" "[ ]" with { } [ ]
		tool_state_json = tool_state_json.replaceAll("\"\\{", "\\{");
		tool_state_json = tool_state_json.replaceAll("\\}\"", "\\}");
		tool_state_json = tool_state_json.replaceAll("\"\\[", "\\[");
		tool_state_json = tool_state_json.replaceAll("\\]\"", "\\]");
		// remove \
		tool_state_json = tool_state_json.replaceAll("\\\\", "");
		// replace "" with "
		tool_state_json = tool_state_json.replaceAll("\"\"", "\"");
		// replace : ", with : "",
		tool_state_json = tool_state_json.replaceAll(": ?\",", ": \"\",");
		// replace UnvalidatedValue with their actual value
		tool_state_json = tool_state_json.replaceAll("\\{\"__class__\":\\s?\"UnvalidatedValue\",\\s?\"value\":\\s?([^\\}]*)\\}", "$1");
		// replace "null" with {"path": ""}
		// this is done such that otpional files will still be accessible by Cheetah as id.path
		tool_state_json = tool_state_json.replaceAll("\"null\"", "{\"path\": \"\"}");
		try {
			this.toolState = new JSONObject(tool_state_json);
		} catch (JSONException e) {
			System.err.println(tool_state_json);
			e.printStackTrace();
			System.exit(-1);
		}
	}

	/**
	 * A method for adding commands to be executed after the task instance (e.g., for moving files from a temporary folder to their destination, as specified in
	 * the tool description)
	 * <commit_msg>Fixed a minor bug that was just introduced<commit_after>/*******************************************************************************
 * In the Hi-WAY project we propose a novel approach of executing scientific
 * workflows processing Big Data, as found in NGS applications, on distributed
 * computational infrastructures. The Hi-WAY software stack comprises the func-
 * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster
 * for Apache Hadoop 2.x (YARN).
 *
 * List of Contributors:
 *
 * <NAME> (HU Berlin)
 * <NAME> (HU Berlin)
 * <NAME> (HU Berlin)
 * <NAME> (HU Berlin)
 *
 * <NAME> is funded by the European Commission through the BiobankCloud
 * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through
 * research training group SOAMED (GRK 1651).
 *
 * Copyright 2014 Humboldt-Universität zu Berlin
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package de.huberlin.wbi.hiway.am.galaxy;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr;
import de.huberlin.wbi.hiway.common.Data;
import de.huberlin.wbi.hiway.common.TaskInstance;

/**
 * An object that extends the Hi-WAY task instance object with its Galaxy tool and tool state, along with methods and structures to utilize and populate them.
 * 
 * @author <NAME>
 *
 */
public class GalaxyTaskInstance extends TaskInstance {
	// the Galaxy tool invoked by this task instance
	private GalaxyTool galaxyTool;
	// input data, which the task instance has to know about for determining the metadata using the Python classes provided by Galaxy
	private Set<GalaxyData> inputs;
	// the Python script run before the actual task instance execution that is responsible for populating the tool state with metadata
	private StringBuilder paramScript;
	// the post script that is to be run subsequent to the actual task invocation and that may involve the moving of files from temporary fodlers to their
	// destination
	private String postScript;
	// the tool state (i.e., the parameter settings in JSON format) of this task instance
	private JSONObject toolState;

	public GalaxyTaskInstance(long id, UUID workflowId, String taskName, GalaxyTool galaxyTool, String galaxyPath) {
		super(id, workflowId, taskName, Math.abs(taskName.hashCode() + 1), ForeignLambdaExpr.LANGID_BASH);
		this.galaxyTool = galaxyTool;
		toolState = new JSONObject();
		paramScript = new StringBuilder("import os, ast\nimport cPickle as pickle\nimport galaxy.app\n");
		this.postScript = "";
		inputs = new HashSet<>();

		// the task instance's invocScript variable is set here for it to be passed to the Worker thread, which writes the script's content as JsonReportEntry
		// to the log
		setInvocScript("script.sh");

		// As opposed to other Hi-WAY applciation masters, the Galaxy AM ha a fairly static command that can be build at task instance creation time
		StringBuilder commandSb = new StringBuilder();
		commandSb.append("PYTHONPATH=" + galaxyPath + "/lib:$PYTHONPATH; export PYTHONPATH\n");
		commandSb.append("PYTHON_EGG_CACHE=.; export PYTHON_EGG_CACHE\n");
		commandSb.append("python params.py\n");
		commandSb.append("cat pre.sh > script.sh\n");
		commandSb.append("echo `cheetah fill template.tmpl --pickle params.p -p` >> script.sh\n");
		commandSb.append("cat post.sh >> script.sh\n");
		commandSb.append("bash ").append(getInvocScript()).append("\n");
		setCommand(commandSb.toString());
	}

	/**
	 * A method that is called when a file name is encountered in the workflow description and the tool state has to be adjusted accordingly
	 * 
	 * @param name
	 *            the parameter name for this file, as specified in the workflow description
	 * @param computeMetadata
	 *            a parameter that determines whether metadata is to be computed for this file, which will only be the case for input data
	 * @param data
	 *            the Hi-WAY data object for this file
	 */
	public void addFile(String name, boolean computeMetadata, GalaxyData data) {
		addFile(name, computeMetadata, data, toolState);
	}

	/**
	 * A (recursive) method that is passed a file along with the JSON object in this task instance's tool state that corresponds to this file. This method is
	 * used to populate the tool state with information on the file.
	 * 
	 * @param name
	 *            the parameter name for this file, as specified in the workflow description
	 * @param computeMetadata
	 *            a parameter that determines whether metadata is to be computed for this file, which will only be the case for input data
	 * @param data
	 *            the Hi-WAY data object for this file
	 * @param jo
	 *            the JSON object in the tool state that corresponds to this file parameter
	 */
	private void addFile(String name, boolean computeMetadata, GalaxyData data, JSONObject jo) {
		try {
			Pattern p = Pattern.compile("(_[0-9]*)?\\|");
			Matcher m = p.matcher(name);

			// (1) if the to-be-added file is part of a repeat (and thus can be identified by an underscore and index number in its name), compute its prefix
			// (the repeat name) as well as its suffix (the actual parameter name) and index; use the prefix and index to obtain its JSON object from the tool
			// state and (recursively) call this method to proceed to (2)
			if (m.find()) {
				String prefix = name.substring(0, m.start());
				String suffix = name.substring(m.end());
				if (m.end() - m.start() > 2) {
					int index = Integer.parseInt(name.substring(m.start() + 1, m.end() - 1));
					JSONArray repeatJa = jo.getJSONArray(prefix);
					for (int i = 0; i < repeatJa.length(); i++) {
						JSONObject repeatJo = repeatJa.getJSONObject(i);
						if (repeatJo.getInt("__index__") == index) {
							addFile(suffix, computeMetadata, data, repeatJo);
							break;
						}
					}
				} else {
					addFile(suffix, computeMetadata, data, jo.getJSONObject(prefix));
				}
				// (2) fix both the template of this tool and the tool state of the task instance calling this method, such that they are in compliance with one
				// another when the tool state is used to set the parameters of the task instance at execution time
			} else {
				// (a) add several properties for this parameter to the tool state
				String fileName = data.getName();
				JSONObject fileJo = new JSONObject();
				fileJo.putOpt("path", fileName);
				fileJo.putOpt("name", fileName.split("\\.(?=[^\\.]+$)")[0]);
				fileJo.putOpt("files_path", data.getLocalDirectory());
				if (data.hasDataType()) {
					GalaxyDataType dataType = data.getDataType();
					String fileExt = dataType.getExtension();
					fileJo.putOpt("extension", fileExt);
					fileJo.putOpt("ext", fileExt);
				}
				// note that this metadata is an empty dictionary that will only be be filled for input data of the task instance calling this method; this is
				// done by executing a python script designed to populate the tool state with metadata prior to execution
				if (data.hasDataType())
					fileJo.putOpt("metadata", new JSONObject());
				jo.putOpt(name, fileJo);

				// (b) adjust the Python script for setting parameters at runtime to compute metadata for this file, given the computeMetadata parameter is set
				if (computeMetadata && data.hasDataType()) {
					GalaxyDataType dataType = data.getDataType();
					paramScript.append("from ");
					paramScript.append(dataType.getFile());
					paramScript.append(" import ");
					paramScript.append(dataType.getName());
					paramScript.append("\n");
					inputs.add(data);
				}
			}
		} catch (JSONException e) {
			e.printStackTrace();
			System.exit(-1);
		}
	}

	/**
	 * A method for setting the tool state of this task instance from the raw string of the workflow description file to the JSON object provided in the Python
	 * parameter script. There are several string operations that have to be performed before the tool state can be interpreted as a JSON object.
	 * 
	 * @param tool_state
	 *            the tool state string, as extracted from the workflow description file
	 * 
	 */
	public void addToolState(String tool_state) {
		String tool_state_json = tool_state;
		// replace "{ }" "[ ]" with { } [ ]
		tool_state_json = tool_state_json.replaceAll("\"\\{", "\\{");
		tool_state_json = tool_state_json.replaceAll("\\}\"", "\\}");
		tool_state_json = tool_state_json.replaceAll("\"\\[", "\\[");
		tool_state_json = tool_state_json.replaceAll("\\]\"", "\\]");
		// remove \
		tool_state_json = tool_state_json.replaceAll("\\\\", "");
		// replace "" with "
		tool_state_json = tool_state_json.replaceAll("\"\"", "\"");
		// replace : ", with : "",
		tool_state_json = tool_state_json.replaceAll(": ?\",", ": \"\",");
		// replace UnvalidatedValue with their actual value
		tool_state_json = tool_state_json.replaceAll("\\{\"__class__\":\\s?\"UnvalidatedValue\",\\s?\"value\":\\s?([^\\}]*)\\}", "$1");
		// replace "null" with {"path": ""}
		// this is done such that otpional files will still be accessible by Cheetah as id.path
		tool_state_json = tool_state_json.replaceAll("\"null\"", "{\"path\": \"\"}");
		try {
			this.toolState = new JSONObject(tool_state_json);
		} catch (JSONException e) {
			System.err.println(tool_state_json);
			e.printStackTrace();
			System.exit(-1);
		}
	}

	/**
	 * A method for adding commands to be executed after the task instance (e.g., for moving files from a temporary folder to their destination, as specified in
	 * the tool description)
	 * 
{'fraction_non_alphanumeric': 0.12035753549065532, 'fraction_numerical': 0.0026767362936762104, 'mean_word_length': 3.7377717391304346, 'pattern_counts': {'":': 12, '<': 22, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6205933', 'n_tokens_mistral': 6409, 'n_tokens_neox': 5745, 'n_words': 2224}