<commit_before>package us.kbase.jgiintegration.perftest;

import static us.kbase.jgiintegration.common.JGIUtils.loadPushableFiles;
import static us.kbase.jgiintegration.common.JGIUtils.wipeRemoteServer;

import java.net.URL;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;

import us.kbase.jgiintegration.common.JGIFileLocation;
import us.kbase.jgiintegration.common.JGIOrganismPage;
import us.kbase.jgiintegration.common.PushableFile;

import com.gargoylesoftware.htmlunit.WebClient;

public class MassPushFiles {
	
	private static final boolean SKIP_WIPE = true;
	private static final String JGI_PUSHABLE_FILES = 
			"/home/crusherofheads/localgit/jgi_kbase_integration_tests/test_data/putative_pushable_files";
	
	private static final int WORKERS = 3;//20;
	private static final int MAX_PUSH_PER_WORKER = 3;

	private static final String WIPE_URL = 
			"http://dev03.berkeley.kbase.us:9000";

	private static String JGI_USER;
	private static String JGI_PWD;
	private static String KB_USER;
	private static String KB_PWD;
	
	
	public static void main(String[] args) throws Exception {
		Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
		JGI_USER = System.getProperty("test.jgi.user");
		JGI_PWD = System.getProperty("test.jgi.pwd");
		KB_USER = System.getProperty("test.kbase.user1");
		KB_PWD = System.getProperty("test.kbase.pwd1");
		
		String wipeUser = System.getProperty("test.kbase.wipe_user");
		String wipePwd = System.getProperty("test.kbase.wipe_pwd");
		if (!SKIP_WIPE) {
			wipeRemoteServer(new URL(WIPE_URL), wipeUser, wipePwd);
		}
		List<PushableFile> files = loadPushableFiles(JGI_PUSHABLE_FILES);
		
		List<List<PushableFile>> filesets =
				new LinkedList<List<PushableFile>>();
		for (int i = 0; i < WORKERS; i++) {
			filesets.add(new LinkedList<PushableFile>());
		}
		int index = 0;
		for (PushableFile pf: files) {
			if (index >= filesets.size()) {
				index = 0;
			}
			filesets.get(index).add(pf);
			index++;
		}

		List<PushFilesToKBaseRunner> theruns =
				new LinkedList<PushFilesToKBaseRunner>();
		for (List<PushableFile> list: filesets) {
			theruns.add(new PushFilesToKBaseRunner(list));
		}
		List<Thread> threads = new LinkedList<Thread>();
		for (PushFilesToKBaseRunner r: theruns) {
			Thread t = new Thread(r);
			t.start();
			threads.add(t);
		}
		
		for (Thread t: threads) {
			t.join();
		}
		
		index = 1;
		int ttlpassed = 0;
		int ttlfailed = 0;
		for (PushFilesToKBaseRunner r: theruns) {
			System.out.println(String.format(
					"Worker %s results:", index,
					r.getResults().size()));
			int passed = 0;
			for (Entry<PushableFile, Throwable> e: r.getResults().entrySet()) {
				PushableFile f = e.getKey();
				String name = f.getOrganism() + "/" + f.getFileGroup() + "/" +
						f.getFile();
				if (e.getValue() == null) {
					System.out.println("\tPushed " + name);
					passed++;
				} else {
					System.out.println("\tException for " + name);
					e.getValue().printStackTrace(System.out);
				}
			}
			System.out.println(String.format("\tPassed: %s, failed: %s",
					passed, r.getResults().size() - passed));
			index++;
			ttlpassed += passed;
			ttlfailed += r.getResults().size() - passed;
		}
		System.out.println(String.format(
				"Total passed: %s, total failed: %s",
				ttlpassed, ttlfailed));
		
	}
	
	private static class PushFilesToKBaseRunner implements Runnable {
		
		private final List<PushableFile> files;
		private final Map<PushableFile, Throwable> results =
				new HashMap<PushableFile, Throwable>();
		public PushFilesToKBaseRunner(List<PushableFile> files) {
			this.files = files;
		}
		
		@Override
		public void run() {
			WebClient wc = new WebClient();
			try {
				//perform known good login
				new JGIOrganismPage(
						wc, "BlaspURHD0036", JGI_USER, JGI_PWD);
			} catch (Exception e) {
				e.printStackTrace();
				return;
			}
			int count = 1;
			for (PushableFile f: files) {
				if (count > MAX_PUSH_PER_WORKER) {
					break;
				}
				results.put(f, null);
				try {
					JGIOrganismPage p = new JGIOrganismPage(
							wc, f.getOrganism(), null, null);
					p.selectFile(new JGIFileLocation(
							f.getFileGroup(), f.getFile()));
					p.pushToKBase(KB_USER, KB_PWD);
				} catch (Exception e) {
					results.put(f, e);
				}
				count++;
			}
		}
		
		public Map<PushableFile, Throwable> getResults() {
			return results;
		}
		<commit_msg>keep results in same order<commit_after>package us.kbase.jgiintegration.perftest;

import static us.kbase.jgiintegration.common.JGIUtils.loadPushableFiles;
import static us.kbase.jgiintegration.common.JGIUtils.wipeRemoteServer;

import java.net.URL;
import java.util.LinkedList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import us.kbase.jgiintegration.common.JGIFileLocation;
import us.kbase.jgiintegration.common.JGIOrganismPage;
import us.kbase.jgiintegration.common.PushableFile;

import com.gargoylesoftware.htmlunit.WebClient;

public class MassPushFiles {
	
	private static final boolean SKIP_WIPE = true;
	private static final String JGI_PUSHABLE_FILES = 
			"/home/crusherofheads/localgit/jgi_kbase_integration_tests/test_data/putative_pushable_files";
	
	private static final int WORKERS = 10;//20;
	private static final int MAX_PUSH_PER_WORKER = 10;

	private static final String WIPE_URL = 
			"http://dev03.berkeley.kbase.us:9000";

	private static String JGI_USER;
	private static String JGI_PWD;
	private static String KB_USER;
	private static String KB_PWD;
	
	
	public static void main(String[] args) throws Exception {
		Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
		JGI_USER = System.getProperty("test.jgi.user");
		JGI_PWD = System.getProperty("test.jgi.pwd");
		KB_USER = System.getProperty("test.kbase.user1");
		KB_PWD = System.getProperty("test.kbase.pwd1");
		
		String wipeUser = System.getProperty("test.kbase.wipe_user");
		String wipePwd = System.getProperty("test.kbase.wipe_pwd");
		if (!SKIP_WIPE) {
			wipeRemoteServer(new URL(WIPE_URL), wipeUser, wipePwd);
		}
		List<PushableFile> files = loadPushableFiles(JGI_PUSHABLE_FILES);
		
		List<List<PushableFile>> filesets =
				new LinkedList<List<PushableFile>>();
		for (int i = 0; i < WORKERS; i++) {
			filesets.add(new LinkedList<PushableFile>());
		}
		int index = 0;
		for (PushableFile pf: files) {
			if (index >= filesets.size()) {
				index = 0;
			}
			filesets.get(index).add(pf);
			index++;
		}

		List<PushFilesToKBaseRunner> theruns =
				new LinkedList<PushFilesToKBaseRunner>();
		for (List<PushableFile> list: filesets) {
			theruns.add(new PushFilesToKBaseRunner(list));
		}
		List<Thread> threads = new LinkedList<Thread>();
		for (PushFilesToKBaseRunner r: theruns) {
			Thread t = new Thread(r);
			t.start();
			threads.add(t);
		}
		
		for (Thread t: threads) {
			t.join();
		}
		
		index = 1;
		int ttlpassed = 0;
		int ttlfailed = 0;
		for (PushFilesToKBaseRunner runner: theruns) {
			System.out.println(String.format(
					"Worker %s results:", index,
					runner.getResults().size()));
			int passed = 0;
			for (Result res: runner.getResults()) {
				PushableFile f = res.file;
				String name = f.getOrganism() + "/" + f.getFileGroup() + "/" +
						f.getFile();
				if (res.exception == null) {
					System.out.println("\tPushed " + name);
					passed++;
				} else {
					System.out.println("\tException for " + name);
					res.exception.printStackTrace(System.out);
				}
			}
			System.out.println(String.format("\tPassed: %s, failed: %s",
					passed, runner.getResults().size() - passed));
			index++;
			ttlpassed += passed;
			ttlfailed += runner.getResults().size() - passed;
		}
		System.out.println(String.format(
				"Total passed: %s, total failed: %s",
				ttlpassed, ttlfailed));
		
	}
	
	private static class Result {
		public PushableFile file;
		public Exception exception;
		public Result(PushableFile file, Exception exception) {
			super();
			this.file = file;
			this.exception = exception;
		}
	}
	
	private static class PushFilesToKBaseRunner implements Runnable {
		
		private final List<PushableFile> files;
		private final List<Result> results =
				new LinkedList<Result>();
		public PushFilesToKBaseRunner(List<PushableFile> files) {
			this.files = files;
		}
		
		@Override
		public void run() {
			WebClient wc = new WebClient();
			try {
				//perform known good login
				new JGIOrganismPage(
						wc, "BlaspURHD0036", JGI_USER, JGI_PWD);
			} catch (Exception e) {
				e.printStackTrace();
				return;
			}
			int count = 1;
			for (PushableFile f: files) {
				if (count > MAX_PUSH_PER_WORKER) {
					break;
				}
				try {
					JGIOrganismPage p = new JGIOrganismPage(
							wc, f.getOrganism(), null, null);
					p.selectFile(new JGIFileLocation(
							f.getFileGroup(), f.getFile()));
					p.pushToKBase(KB_USER, KB_PWD);
					results.add(new Result(f, null));
				} catch (Exception e) {
					results.add(new Result(f, e));
				}
				count++;
			}
		}
		
		public List<Result> getResults() {
			return results;
		}
		
{'fraction_non_alphanumeric': 0.1295365107127241, 'fraction_numerical': 0.005465675557498907, 'mean_word_length': 3.7135497166409066, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6198984', 'n_tokens_mistral': 3778, 'n_tokens_neox': 3146, 'n_words': 660}