<commit_before>package org.ossmeter.platform.osgi.executors;

import java.io.FileWriter;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import org.ossmeter.platform.Date;
import org.ossmeter.platform.IMetricProvider;
import org.ossmeter.platform.Platform;
import org.ossmeter.platform.delta.NoManagerFoundException;
import org.ossmeter.platform.delta.ProjectDelta;
import org.ossmeter.platform.logging.OssmeterLogger;
import org.ossmeter.repository.model.BugTrackingSystem;
import org.ossmeter.repository.model.CommunicationChannel;
import org.ossmeter.repository.model.Project;
import org.ossmeter.repository.model.ProjectExecutionInformation;
import org.ossmeter.repository.model.VcsRepository;

public class ProjectExecutor implements Runnable {
	
	protected FileWriter writer;

	protected Project project;
	protected int numberOfCores;
	protected Platform platform;
	protected OssmeterLogger logger;
	
	public ProjectExecutor(Platform platform, Project project) {
		this.numberOfCores = Runtime.getRuntime().availableProcessors();
		this.platform = platform;
		this.project = project;
		this.logger = (OssmeterLogger)OssmeterLogger.getLogger("ProjectExecutor (" + project.getName() +")");
		this.logger.addConsoleAppender(OssmeterLogger.DEFAULT_PATTERN);
		
		// DEBUG
		try {
//			this.writer = null;
			this.writer = new FileWriter("/Users/jimmy/Desktop/D5.3-logs/" + project.getName() + ".csv");
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	@Override
	public void run() {
		if (project == null) {
			logger.error("No project scheduled. Exiting.");
			return;
		}
		logger.info("Beginning execution.");
		
		// Clear any open flags
		if (project.getExecutionInformation() == null) {
			project.setExecutionInformation(new ProjectExecutionInformation());
		}
		project.getExecutionInformation().setInErrorState(false);
		platform.getProjectRepositoryManager().getProjectRepository().sync();
		
		// Split metrics into branches
		List<List<IMetricProvider>> metricBranches = splitIntoBranches(platform.getMetricProviderManager().getMetricProviders());
		
		// Find the date to start from 
		Date lastExecuted = getLastExecutedDate();
		
		if (lastExecuted == null) {
			// TODO: Perhaps flag the project as being in a fatal error state? This will potentially keep occurring.
			logger.error("Parse error of project's lastExecuted date. Returned null.");
			return;
		}
		Date today = new Date();
		
		Date[] dates = Date.range(lastExecuted.addDays(1), today.addDays(-1));
		
		
		for (Date date : dates) {
			// TODO: An alternative would be to have a single thread pool for the node. I briefly tried this
			// and it didn't work. Reverted to this implement (temporarily at least).
			ExecutorService executorService = Executors.newFixedThreadPool(numberOfCores);
			logger.info("Date: " + date + ", project: " + project.getName());
			
			long startDelta = System.currentTimeMillis();
			ProjectDelta delta = new ProjectDelta(project, date, platform);
			boolean createdOk = delta.create();
			String deltaTimes = delta.getTimingsString();
			long timeDelta = System.currentTimeMillis() - startDelta;

			if (createdOk) {
			} else {
				project.getExecutionInformation().setInErrorState(true);
				platform.getProjectRepositoryManager().getProjectRepository().sync();
				
				logger.error("Project delta creation failed. Aborting.");
				return;
			}
			
			long startMetrics = System.currentTimeMillis();
			for (List<IMetricProvider> branch : metricBranches) {
				MetricListExecutor mExe = new MetricListExecutor(platform, project, delta, date);
				mExe.setMetricList(branch);
				
				executorService.execute(mExe);
			}
			
			try {
				executorService.shutdown();
				executorService.awaitTermination(24, TimeUnit.HOURS);
			} catch (InterruptedException e) {
				logger.error("Exception thrown when shutting down executor service.", e);
			}
			long timeMetrics = System.currentTimeMillis() - startMetrics;
			
			if (project.getExecutionInformation().getInErrorState()) {
				// TODO: what should we do? Is the act of not-updating the lastExecuted flag enough?
				// If it continues to loop, it simply tries tomorrow. We need to stop this happening.
				logger.warn("Project in error state. Stopping execution.");
				break;
			} else {
				project.getExecutionInformation().setLastExecuted(date.toString());
				platform.getProjectRepositoryManager().getProjectRepository().sync();
			}
			
			try { ///DEBUG
				writer.write(date.toString() + "," + deltaTimes + "," + timeDelta + "," + timeMetrics + "\n");
				writer.flush();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		try {
			writer.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		logger.info("Project execution complete. In error state: " + project.getExecutionInformation().getInErrorState());
	}

	/**
	 * Algorithm to split a list of metric providers into dependency branches. Current implementation isn't
	 * wonderful - it was built to work, not to perform - and needs relooking at in the future. 
	 * @param metrics
	 * @return
	 */
	public List<List<IMetricProvider>> splitIntoBranches(List<IMetricProvider> metrics) {
		List<Set<IMetricProvider>> branches = new ArrayList<Set<IMetricProvider>>();
		
		for (IMetricProvider m : metrics) {
			Set<IMetricProvider> mBranch = new HashSet<>();
			
			for (Set<IMetricProvider> branch : branches) {
				if (branch.contains(m)) {
					mBranch = branch;
					break;
				}
			}
			if (!mBranch.contains(m)) mBranch.add(m);
			if (!branches.contains(mBranch)) branches.add(mBranch);

			for (String id : m.getIdentifiersOfUses()) {
				IMetricProvider use = lookupMetricProviderById(metrics, id);
				if (use == null) continue;
				boolean foundUse = false;
				for (Set<IMetricProvider> branch : branches) {
					if (branch.contains(use)) {
						branch.addAll(mBranch);
						branches.remove(mBranch);
						mBranch = branch;
						foundUse = true;
						break;<commit_msg>Project local storage path added.<commit_after>package org.ossmeter.platform.osgi.executors;

import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import org.ossmeter.platform.Date;
import org.ossmeter.platform.IMetricProvider;
import org.ossmeter.platform.Platform;
import org.ossmeter.platform.delta.NoManagerFoundException;
import org.ossmeter.platform.delta.ProjectDelta;
import org.ossmeter.platform.logging.OssmeterLogger;
import org.ossmeter.repository.model.BugTrackingSystem;
import org.ossmeter.repository.model.CommunicationChannel;
import org.ossmeter.repository.model.LocalStorage;
import org.ossmeter.repository.model.Project;
import org.ossmeter.repository.model.ProjectExecutionInformation;
import org.ossmeter.repository.model.VcsRepository;

public class ProjectExecutor implements Runnable {
	
	protected Project project;
	protected int numberOfCores;
	protected Platform platform;
	protected OssmeterLogger logger;
	
	public ProjectExecutor(Platform platform, Project project) {
		this.numberOfCores = Runtime.getRuntime().availableProcessors();
		this.platform = platform;
		this.project = project;
		this.logger = (OssmeterLogger)OssmeterLogger.getLogger("ProjectExecutor (" + project.getName() +")");
		this.logger.addConsoleAppender(OssmeterLogger.DEFAULT_PATTERN);
		
	}
	
	protected void initialiseProjectLocalStorage (Project project) {
		try{	
			Path projectLocalStoragePath = Paths.get(platform.getLocalStorageHomeDirectory().toString(), project.getName());		
			if (Files.notExists(projectLocalStoragePath)) {
				Files.createDirectory(projectLocalStoragePath);
			}
			LocalStorage projectLocalStorage = new LocalStorage();
			projectLocalStorage.setPath(projectLocalStoragePath.toString());
			project.getExecutionInformation().setStorage(projectLocalStorage);
		} catch(IOException e) {
			e.printStackTrace();
		}
	}
	
	@Override
	public void run() {
		if (project == null) {
			logger.error("No project scheduled. Exiting.");
			return;
		}
		logger.info("Beginning execution.");
		
		initialiseProjectLocalStorage(project);
		
		// Clear any open flags
		if (project.getExecutionInformation() == null) {
			project.setExecutionInformation(new ProjectExecutionInformation());
		}
		project.getExecutionInformation().setInErrorState(false);
		platform.getProjectRepositoryManager().getProjectRepository().sync();
		
		// Split metrics into branches
		List<List<IMetricProvider>> metricBranches = splitIntoBranches(platform.getMetricProviderManager().getMetricProviders());
		
		// Find the date to start from 
		Date lastExecuted = getLastExecutedDate();
		
		if (lastExecuted == null) {
			// TODO: Perhaps flag the project as being in a fatal error state? This will potentially keep occurring.
			logger.error("Parse error of project's lastExecuted date. Returned null.");
			return;
		}
		Date today = new Date();
		
		Date[] dates = Date.range(lastExecuted.addDays(1), today.addDays(-1));
		
		
		for (Date date : dates) {
			// TODO: An alternative would be to have a single thread pool for the node. I briefly tried this
			// and it didn't work. Reverted to this implement (temporarily at least).
			ExecutorService executorService = Executors.newFixedThreadPool(numberOfCores);
			logger.info("Date: " + date + ", project: " + project.getName());
			
			long startDelta = System.currentTimeMillis();
			ProjectDelta delta = new ProjectDelta(project, date, platform);
			boolean createdOk = delta.create();
			String deltaTimes = delta.getTimingsString();
			long timeDelta = System.currentTimeMillis() - startDelta;

			if (createdOk) {
			} else {
				project.getExecutionInformation().setInErrorState(true);
				platform.getProjectRepositoryManager().getProjectRepository().sync();
				
				logger.error("Project delta creation failed. Aborting.");
				return;
			}
			
			long startMetrics = System.currentTimeMillis();
			for (List<IMetricProvider> branch : metricBranches) {
				MetricListExecutor mExe = new MetricListExecutor(platform, project, delta, date);
				mExe.setMetricList(branch);
				
				executorService.execute(mExe);
			}
			
			try {
				executorService.shutdown();
				executorService.awaitTermination(24, TimeUnit.HOURS);
			} catch (InterruptedException e) {
				logger.error("Exception thrown when shutting down executor service.", e);
			}
			long timeMetrics = System.currentTimeMillis() - startMetrics;
			
			if (project.getExecutionInformation().getInErrorState()) {
				// TODO: what should we do? Is the act of not-updating the lastExecuted flag enough?
				// If it continues to loop, it simply tries tomorrow. We need to stop this happening.
				logger.warn("Project in error state. Stopping execution.");
				break;
			} else {
				project.getExecutionInformation().setLastExecuted(date.toString());
				platform.getProjectRepositoryManager().getProjectRepository().sync();
			}
			
		}
		logger.info("Project execution complete. In error state: " + project.getExecutionInformation().getInErrorState());
	}

	/**
	 * Algorithm to split a list of metric providers into dependency branches. Current implementation isn't
	 * wonderful - it was built to work, not to perform - and needs relooking at in the future. 
	 * @param metrics
	 * @return
	 */
	public List<List<IMetricProvider>> splitIntoBranches(List<IMetricProvider> metrics) {
		List<Set<IMetricProvider>> branches = new ArrayList<Set<IMetricProvider>>();
		
		for (IMetricProvider m : metrics) {
			Set<IMetricProvider> mBranch = new HashSet<>();
			
			for (Set<IMetricProvider> branch : branches) {
				if (branch.contains(m)) {
					mBranch = branch;
					break;
				}
			}
			if (!mBranch.contains(m)) mBranch.add(m);
			if (!branches.contains(mBranch)) branches.add(mBranch);

			//FIXME: Test m.getIdentifiersOfUses() for null
			
			for (String id : m.getIdentifiersOfUses()) {
				IMetricProvider use = lookupMetricProviderById(metrics, id);
				if (use == null) continue;
				boolean foundUse = false;
				for (Set<IMetricProvider> branch : branches) {
					if (branch.contains(use)) {
						branch.addAll(mBranch);
						branches.remove(mBranch);
						mBranch = branch;
						foundUse = true;
						break;
{'fraction_non_alphanumeric': 0.11178557351202208, 'fraction_numerical': 0.0007883326763894363, 'mean_word_length': 4.319077568134172, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1924286', 'n_tokens_mistral': 4157, 'n_tokens_neox': 3528, 'n_words': 871}