<commit_before>/*******************************************************************************
 * DIANNE  - Framework for distributed artificial neural networks
 * Copyright (C) 2015  iMinds - IBCN - UGent
 *
 * This file is part of DIANNE.
 *
 * DIANNE is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Contributors:
 *     <NAME>, <NAME>
 *******************************************************************************/
package be.iminds.iot.dianne.nn.eval;

import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.osgi.framework.BundleContext;
import org.osgi.framework.Constants;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.osgi.service.component.annotations.ReferencePolicy;

import be.iminds.iot.dianne.api.dataset.Batch;
import be.iminds.iot.dianne.api.dataset.Dataset;
import be.iminds.iot.dianne.api.dataset.DianneDatasets;
import be.iminds.iot.dianne.api.dataset.Sample;
import be.iminds.iot.dianne.api.log.DataLogger;
import be.iminds.iot.dianne.api.nn.Dianne;
import be.iminds.iot.dianne.api.nn.NeuralNetwork;
import be.iminds.iot.dianne.api.nn.eval.Evaluation;
import be.iminds.iot.dianne.api.nn.eval.EvaluationProgress;
import be.iminds.iot.dianne.api.nn.eval.Evaluator;
import be.iminds.iot.dianne.api.nn.eval.EvaluatorListener;
import be.iminds.iot.dianne.api.nn.module.Module.Mode;
import be.iminds.iot.dianne.api.nn.module.dto.NeuralNetworkInstanceDTO;
import be.iminds.iot.dianne.nn.eval.config.EvaluatorConfig;
import be.iminds.iot.dianne.nn.util.DianneConfigHandler;
import be.iminds.iot.dianne.tensor.Tensor;

/**
 * The AbstractEvaluator has all mechanics to loop through (part of) a Dataset
 * and forward each sample.
 * 
 * Concrete evaluators should implement the evalOutput(index, output, expected output)
 * and update the error/confusionMatrix
 * @author tverbele
 *
 */
public abstract class AbstractEvaluator implements Evaluator {
	
	private static ExecutorService listenerExecutor = Executors.newSingleThreadExecutor(); 
	protected List<EvaluatorListener> listeners = Collections.synchronizedList(new ArrayList<>());
	
	protected UUID evaluatorId;
	
	protected DataLogger logger;
	
	protected Dianne dianne;
	protected DianneDatasets datasets;
	
	protected EvaluatorConfig config;
	
	protected volatile boolean evaluating = false;
	
	protected int index = 0;
	protected int total;
	protected float error;
	protected List<Tensor> outputs;
	protected long tStart, tEnd, tForward;
	
	@Override
	public UUID getEvaluatorId(){
		return evaluatorId;
	}

	@Override
	public synchronized Evaluation eval(String dataset,
			Map<String, String> config, NeuralNetworkInstanceDTO nni) throws Exception {
		if(evaluating){
			throw new Exception("Already running an evaluation session here");
		}
		evaluating = true;
		
		Dataset d = null;
		try {
			System.out.println("Evaluator Configuration");
			System.out.println("=======================");
			
			this.config = DianneConfigHandler.getConfig(config, EvaluatorConfig.class);

			// Fetch the dataset
			d = datasets.configureDataset(dataset, config);
			if(d==null){
				throw new Exception("Dataset "+dataset+" not available");
			}
			total = d.size();
			error = 0.0f;
			init(config);
			
			NeuralNetwork nn = null;
			try {
				nn = dianne.getNeuralNetwork(nni).getValue();
			} catch (Exception e) {
				throw new Exception("Neural Network "+nni.id+" not available!");
			}
			nn.getModules().values().stream().forEach(m -> m.setMode(EnumSet.of(Mode.BLOCKING)));
			
			try {
				if(this.config.tag==null){
					nn.loadParameters();
				} else {
					nn.loadParameters(this.config.tag);
				}
			} catch(Exception e){
				// ignore if no parameters found
				System.out.println("No parameters loaded for this evaluation - network is not yet trained?");
			}
		
			outputs = this.config.includeOutputs ? new ArrayList<Tensor>() : null;
			tStart = System.currentTimeMillis(); tForward = 0;
			
			Sample sample = null;
			Batch batch = null;
			int[] indices = null;
			boolean batchMode = false;
			if(this.config.batchSize > 0 && d.inputDims() != null){
				batchMode = true;
				indices = new int[this.config.batchSize];
			}
			
			for(index=0;index<total;index++){
				Tensor out;
				if(batchMode){<commit_msg>adjust final batch size in evaluation<commit_after>/*******************************************************************************
 * DIANNE  - Framework for distributed artificial neural networks
 * Copyright (C) 2015  iMinds - IBCN - UGent
 *
 * This file is part of DIANNE.
 *
 * DIANNE is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Contributors:
 *     <NAME>, <NAME>
 *******************************************************************************/
package be.iminds.iot.dianne.nn.eval;

import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.osgi.framework.BundleContext;
import org.osgi.framework.Constants;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.osgi.service.component.annotations.ReferencePolicy;

import be.iminds.iot.dianne.api.dataset.Batch;
import be.iminds.iot.dianne.api.dataset.Dataset;
import be.iminds.iot.dianne.api.dataset.DianneDatasets;
import be.iminds.iot.dianne.api.dataset.Sample;
import be.iminds.iot.dianne.api.log.DataLogger;
import be.iminds.iot.dianne.api.nn.Dianne;
import be.iminds.iot.dianne.api.nn.NeuralNetwork;
import be.iminds.iot.dianne.api.nn.eval.Evaluation;
import be.iminds.iot.dianne.api.nn.eval.EvaluationProgress;
import be.iminds.iot.dianne.api.nn.eval.Evaluator;
import be.iminds.iot.dianne.api.nn.eval.EvaluatorListener;
import be.iminds.iot.dianne.api.nn.module.Module.Mode;
import be.iminds.iot.dianne.api.nn.module.dto.NeuralNetworkInstanceDTO;
import be.iminds.iot.dianne.nn.eval.config.EvaluatorConfig;
import be.iminds.iot.dianne.nn.util.DianneConfigHandler;
import be.iminds.iot.dianne.tensor.Tensor;

/**
 * The AbstractEvaluator has all mechanics to loop through (part of) a Dataset
 * and forward each sample.
 * 
 * Concrete evaluators should implement the evalOutput(index, output, expected output)
 * and update the error/confusionMatrix
 * @author tverbele
 *
 */
public abstract class AbstractEvaluator implements Evaluator {
	
	private static ExecutorService listenerExecutor = Executors.newSingleThreadExecutor(); 
	protected List<EvaluatorListener> listeners = Collections.synchronizedList(new ArrayList<>());
	
	protected UUID evaluatorId;
	
	protected DataLogger logger;
	
	protected Dianne dianne;
	protected DianneDatasets datasets;
	
	protected EvaluatorConfig config;
	
	protected volatile boolean evaluating = false;
	
	protected int index = 0;
	protected int total;
	protected float error;
	protected List<Tensor> outputs;
	protected long tStart, tEnd, tForward;
	
	@Override
	public UUID getEvaluatorId(){
		return evaluatorId;
	}

	@Override
	public synchronized Evaluation eval(String dataset,
			Map<String, String> config, NeuralNetworkInstanceDTO nni) throws Exception {
		if(evaluating){
			throw new Exception("Already running an evaluation session here");
		}
		evaluating = true;
		
		Dataset d = null;
		try {
			System.out.println("Evaluator Configuration");
			System.out.println("=======================");
			
			this.config = DianneConfigHandler.getConfig(config, EvaluatorConfig.class);

			// Fetch the dataset
			d = datasets.configureDataset(dataset, config);
			if(d==null){
				throw new Exception("Dataset "+dataset+" not available");
			}
			total = d.size();
			error = 0.0f;
			init(config);
			
			NeuralNetwork nn = null;
			try {
				nn = dianne.getNeuralNetwork(nni).getValue();
			} catch (Exception e) {
				throw new Exception("Neural Network "+nni.id+" not available!");
			}
			nn.getModules().values().stream().forEach(m -> m.setMode(EnumSet.of(Mode.BLOCKING)));
			
			try {
				if(this.config.tag==null){
					nn.loadParameters();
				} else {
					nn.loadParameters(this.config.tag);
				}
			} catch(Exception e){
				// ignore if no parameters found
				System.out.println("No parameters loaded for this evaluation - network is not yet trained?");
			}
		
			outputs = this.config.includeOutputs ? new ArrayList<Tensor>() : null;
			tStart = System.currentTimeMillis(); tForward = 0;
			
			Sample sample = null;
			Batch batch = null;
			int[] indices = null;
			boolean batchMode = false;
			if(this.config.batchSize > 0 && d.inputDims() != null){
				batchMode = true;
				indices = new int[this.config.batchSize];
			}
			
			for(index=0;index<total;index++){
				Tensor out;
				if(batchMode){
					// for last batch, adjust to remaining size
					if(index+indices.length > total){
						indices = new int[total-index];
						batch = null;
					}
{'fraction_non_alphanumeric': 0.13778287023775423, 'fraction_numerical': 0.002100639740284541, 'mean_word_length': 4.279233870967742, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5632333', 'n_tokens_mistral': 3491, 'n_tokens_neox': 3140, 'n_words': 826}