<commit_before>package be.iminds.iot.dianne.coordinator;

import java.util.HashMap;
import java.util.Map;
import java.util.UUID;

import be.iminds.iot.dianne.api.coordinator.EvaluationResult;
import be.iminds.iot.dianne.api.coordinator.Job.Type;
import be.iminds.iot.dianne.api.nn.eval.Evaluation;
import be.iminds.iot.dianne.api.nn.eval.EvaluationProgress;
import be.iminds.iot.dianne.api.nn.eval.Evaluator;
import be.iminds.iot.dianne.api.nn.module.dto.NeuralNetworkDTO;

public class EvaluationJob extends AbstractJob<EvaluationResult> {

	private EvaluationResult result = null;
	
	public EvaluationJob(DianneCoordinatorImpl coord,
			NeuralNetworkDTO nn,
			String d,
			Map<String, String> c){
		super(coord, Type.EVALUATE, nn, d, c);
	}
	
	@Override
	public void execute() throws Exception {
		
		Map<String, String> evalConfig = new HashMap<>(config);
		if(!evalConfig.containsKey("range")){
			evalConfig.put("range", config.get("testSet"));
		}

		Map<UUID, Evaluation> results = new HashMap<>();
		for(UUID target : targets){
			// TODO in parallel?
			Evaluator evaluator = coordinator.evaluators.get(target);
			Evaluation e = evaluator.eval(nnis.get(target), dataset, evalConfig);
			results.put(target, e);
		}
		
		result = new EvaluationResult(results);
		
		
		done(result);
	}

	@Override
	public EvaluationResult getProgress() {
		if(result!=null)
			return result;
		
		Map<UUID, Evaluation> results = new HashMap<>();
		for(UUID target : targets){
			Evaluator evaluator = coordinator.evaluators.get(target);
			EvaluationProgress e = evaluator.getProgress();
			results.put(target, e);
		}
		return new EvaluationResult(results);
	}

}
<commit_msg>run multiple evaluations on separate threads<commit_after>package be.iminds.iot.dianne.coordinator;

import java.util.HashMap;
import java.util.Map;
import java.util.UUID;

import be.iminds.iot.dianne.api.coordinator.EvaluationResult;
import be.iminds.iot.dianne.api.coordinator.Job.Type;
import be.iminds.iot.dianne.api.nn.eval.Evaluation;
import be.iminds.iot.dianne.api.nn.eval.EvaluationProgress;
import be.iminds.iot.dianne.api.nn.eval.Evaluator;
import be.iminds.iot.dianne.api.nn.module.dto.NeuralNetworkDTO;

public class EvaluationJob extends AbstractJob<EvaluationResult> {

	private EvaluationResult result = null;
	
	public EvaluationJob(DianneCoordinatorImpl coord,
			NeuralNetworkDTO nn,
			String d,
			Map<String, String> c){
		super(coord, Type.EVALUATE, nn, d, c);
	}
	
	@Override
	public void execute() throws Exception {
		
		Map<String, String> evalConfig = new HashMap<>(config);
		if(!evalConfig.containsKey("range")){
			evalConfig.put("range", config.get("testSet"));
		}

		final Map<UUID, Evaluation> results = new HashMap<>();
		Thread[] threads = new Thread[targets.size()];
		for(int i=0;i<targets.size();i++){
			final UUID target = targets.get(i);
			threads[i] = new Thread(new Runnable(){
				public void run(){
					try {
						Evaluator evaluator = coordinator.evaluators.get(target);
						Evaluation e = evaluator.eval(nnis.get(target), dataset, evalConfig);
						results.put(target, e);
					} catch(Exception e){
						// TODO should this fail the entire job?
						results.put(target, null);
					}
				}
			});
			threads[i].start();
		}
		
		for(Thread t : threads){
			t.join();
		}
		
		result = new EvaluationResult(results);
		
		done(result);
	}

	@Override
	public EvaluationResult getProgress() {
		if(result!=null)
			return result;
		
		Map<UUID, Evaluation> results = new HashMap<>();
		for(UUID target : targets){
			Evaluator evaluator = coordinator.evaluators.get(target);
			EvaluationProgress e = evaluator.getProgress();
			results.put(target, e);
		}
		return new EvaluationResult(results);
	}

}

{'fraction_non_alphanumeric': 0.13083109919571045, 'fraction_numerical': 0.00026809651474530834, 'mean_word_length': 4.284702549575071, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3320654', 'n_tokens_mistral': 1409, 'n_tokens_neox': 1226, 'n_words': 252}