<commit_before>package org.griphyn.vdl.karajan.lib;

import org.globus.cog.karajan.arguments.Arg;
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection;
import org.globus.cog.karajan.util.TypeUtil;

import java.util.Map;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.ArrayList;

/** this is an icky class that does too much with globals, but is for
proof of concept. */

public class RuntimeStats extends FunctionsCollection {

	public static final Arg PA_STATE = new Arg.Positional("state");
	public static final int MIN_PERIOD_MS=5000;
	public static final int MAX_PERIOD_MS=60000;

	static {
		setArguments("vdl_startprogressticker", new Arg[0]);
		setArguments("vdl_stopprogressticker", new Arg[0]);
		setArguments("vdl_setprogress", new Arg[] { PA_STATE } );
		setArguments("vdl_initprogressstate", new Arg[] { PA_STATE });
	}

	public Object vdl_startprogressticker(VariableStack stack) throws ExecutionException {
		ProgressTicker t = new ProgressTicker();
		t.setDaemon(true);
		t.start();
		stack.parentFrame().setVar("#swift-runtime-progress-ticker",t);
		return null;
	}


	public Object vdl_setprogress(VariableStack stack) throws ExecutionException {
		RuntimeProgress rp = (RuntimeProgress)stack.getVar("#swift-runtime-progress");
		rp.status = TypeUtil.toString(PA_STATE.getValue(stack));
		ProgressTicker p = (ProgressTicker)stack.getVar("#swift-runtime-progress-ticker");
		p.dumpState();
		return null;
	}

	public Object vdl_initprogressstate(VariableStack stack) throws ExecutionException {
		RuntimeProgress rp = new RuntimeProgress();
		ProgressTicker p = (ProgressTicker)stack.getVar("#swift-runtime-progress-ticker");
		p.states.add(rp);
		stack.parentFrame().setVar("#swift-runtime-progress",rp);
		p.dumpState();
		return null;
	}

	public synchronized Object vdl_stopprogressticker(VariableStack stack) throws ExecutionException {
		ProgressTicker p = (ProgressTicker)stack.getVar("#swift-runtime-progress-ticker");
		p.finalDumpState();
		p.stop();
		return null;
	}


	static public class ProgressTicker extends Thread {

		List states = new ArrayList();

		long lastDumpTime = 0;

		public ProgressTicker() {
			super("Progress ticker");
		}

		public void run() {
			while(true) {
				dumpState();

				try {
					Thread.sleep(MAX_PERIOD_MS);
				} catch(InterruptedException e) {

					System.err.println("Runtime ticker interrupted. Looping immediately.");
				}
			}
		}

		void dumpState() {
			long now = System.currentTimeMillis();
			if(lastDumpTime + MIN_PERIOD_MS > now) return;
			lastDumpTime = now;
			printStates("Progress: ");
		}

		void finalDumpState() {
			printStates("Final status: ");
		}

		void printStates(String header) {
			Map summary = new HashMap();
			Iterator stateIterator = states.iterator();

			// summarize details of known states into summary, with
			// one entry per state type, storing the number of
			// jobs in that state.
			while(stateIterator.hasNext()) {
				String key = ((RuntimeProgress)stateIterator.next()).status;
				Integer count = (Integer) summary.get(key);
				if(count == null) {
					summary.put(key,new Integer(1));
				} else {
					summary.put(key,new Integer(count.intValue()+1));
				}
			}

			// output the results of summarization, in a relatively
			// pretty form
			System.err.print(header);
			Iterator summaryIterator = summary.keySet().iterator();
			while(summaryIterator.hasNext()) {
				Object o = summaryIterator.next();
				System.err.print(" "+o+":"+summary.get(o));
			}
			System.err.println("");
		}


	}

	class RuntimeProgress {
		String status = "uninitialized";
	}

}<commit_msg>runtime progress ticker outputs in a more logical order based on the semantics of execution rather than the vagaries of map implementation<commit_after>package org.griphyn.vdl.karajan.lib;

import org.globus.cog.karajan.arguments.Arg;
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection;
import org.globus.cog.karajan.util.TypeUtil;

import java.util.Map;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.ArrayList;

/** this is an icky class that does too much with globals, but is for
proof of concept. */

public class RuntimeStats extends FunctionsCollection {

	public static final Arg PA_STATE = new Arg.Positional("state");
	public static final int MIN_PERIOD_MS=5000;
	public static final int MAX_PERIOD_MS=60000;

	public static final String[] preferredOutputOrder = {
		"Initializing",
		"Selecting site",
		"Stage in",
		"Executing",
		"Stage out",
		"Failed",
		"Finished successfully"
	};

	static {
		setArguments("vdl_startprogressticker", new Arg[0]);
		setArguments("vdl_stopprogressticker", new Arg[0]);
		setArguments("vdl_setprogress", new Arg[] { PA_STATE } );
		setArguments("vdl_initprogressstate", new Arg[] { PA_STATE });
	}

	public Object vdl_startprogressticker(VariableStack stack) throws ExecutionException {
		ProgressTicker t = new ProgressTicker();
		t.setDaemon(true);
		t.start();
		stack.parentFrame().setVar("#swift-runtime-progress-ticker",t);
		return null;
	}


	public Object vdl_setprogress(VariableStack stack) throws ExecutionException {
		RuntimeProgress rp = (RuntimeProgress)stack.getVar("#swift-runtime-progress");
		rp.status = TypeUtil.toString(PA_STATE.getValue(stack));
		ProgressTicker p = (ProgressTicker)stack.getVar("#swift-runtime-progress-ticker");
		p.dumpState();
		return null;
	}

	public Object vdl_initprogressstate(VariableStack stack) throws ExecutionException {
		RuntimeProgress rp = new RuntimeProgress();
		ProgressTicker p = (ProgressTicker)stack.getVar("#swift-runtime-progress-ticker");
		p.states.add(rp);
		stack.parentFrame().setVar("#swift-runtime-progress",rp);
		p.dumpState();
		return null;
	}

	public synchronized Object vdl_stopprogressticker(VariableStack stack) throws ExecutionException {
		ProgressTicker p = (ProgressTicker)stack.getVar("#swift-runtime-progress-ticker");
		p.finalDumpState();
		p.stop();
		return null;
	}


	static public class ProgressTicker extends Thread {

		List states = new ArrayList();

		long lastDumpTime = 0;

		public ProgressTicker() {
			super("Progress ticker");
		}

		public void run() {
			while(true) {
				dumpState();

				try {
					Thread.sleep(MAX_PERIOD_MS);
				} catch(InterruptedException e) {

					System.err.println("Runtime ticker interrupted. Looping immediately.");
				}
			}
		}

		void dumpState() {
			long now = System.currentTimeMillis();
			if(lastDumpTime + MIN_PERIOD_MS > now) return;
			lastDumpTime = now;
			printStates("Progress: ");
		}

		void finalDumpState() {
			printStates("Final status: ");
		}

		void printStates(String header) {
			Map summary = new HashMap();
			Iterator stateIterator = states.iterator();

			// summarize details of known states into summary, with
			// one entry per state type, storing the number of
			// jobs in that state.
			while(stateIterator.hasNext()) {
				String key = ((RuntimeProgress)stateIterator.next()).status;
				Integer count = (Integer) summary.get(key);
				if(count == null) {
					summary.put(key,new Integer(1));
				} else {
					summary.put(key,new Integer(count.intValue()+1));
				}
			}

			// output the results of summarization, in a relatively
			// pretty form - first the preferred order listed elements,
			// and then anything remaining
			System.err.print(header);

			for(int pos = 0; pos < preferredOutputOrder.length; pos++) {
				String key = preferredOutputOrder[pos];
				Object value = summary.get(key);
				if(value != null)
					System.err.print(" "+key+":"+value);
				summary.remove(key);
			}

			Iterator summaryIterator = summary.keySet().iterator();
			while(summaryIterator.hasNext()) {
				Object o = summaryIterator.next();
				System.err.print(" "+o+":"+summary.get(o));
			}
			System.err.println("");
		}

	}

	class RuntimeProgress {
		String status = "uninitialized";
	}

}
{'fraction_non_alphanumeric': 0.12295081967213115, 'fraction_numerical': 0.003547834597504282, 'mean_word_length': 4.339647289353364, 'pattern_counts': {'":': 3, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3428228', 'n_tokens_mistral': 2877, 'n_tokens_neox': 2505, 'n_words': 582}