<commit_before>package org.metaborg.runtime.task;

import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.spoofax.interpreter.terms.IStrategoInt;
import org.spoofax.interpreter.terms.IStrategoList;
import org.spoofax.interpreter.terms.IStrategoString;
import org.spoofax.interpreter.terms.IStrategoTerm;
import org.spoofax.interpreter.terms.IStrategoTuple;
import org.spoofax.interpreter.terms.ITermFactory;

import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Multimap;

public class Tasks {
	private final ITermFactory factory;
	private final ManyToManyMap<IStrategoInt, IStrategoString> toPartition = ManyToManyMap.create();
	private final ManyToManyMap<IStrategoInt, IStrategoInt> toDependency = ManyToManyMap.create();
	private final Map<IStrategoInt, IStrategoTerm> toInstruction = new ConcurrentHashMap<IStrategoInt, IStrategoTerm>();
	private final Map<IStrategoInt, IStrategoList> toResult = new ConcurrentHashMap<IStrategoInt, IStrategoList>();
	
	private final Set<IStrategoInt> initialTasks = new HashSet<IStrategoInt>();
	private final Set<IStrategoInt> addedTasks = new HashSet<IStrategoInt>();
	
	public Tasks(ITermFactory factory) {
		this.factory = factory;
	}
	
	public IStrategoInt addTask(IStrategoTerm instruction, IStrategoString partition, IStrategoList dependencies) {
		
		IStrategoInt taskID = factory.makeInt(instruction.hashCode());
		if(toInstruction.put(taskID, instruction) == null)
			addedTasks.add(taskID);
		initialTasks.remove(taskID);
		
		toPartition.put(taskID, partition);
		
        while(!dependencies.isEmpty()) {
        	IStrategoInt head = (IStrategoInt) dependencies.head();
            toDependency.put(taskID, head);
            dependencies = dependencies.tail();
        }
        
        return taskID;
	}
	
	public void addResult(IStrategoInt taskID, IStrategoTerm result) {
		toResult.put(taskID, factory.makeListCons(result, factory.makeList()));
	}
	
	public void addResult(IStrategoInt taskID, IStrategoList resultList) {
		toResult.put(taskID, resultList);
	}
	
	public void startAnalysis(IStrategoString partition) {
		addedTasks.clear();
		initialTasks.clear();
		initialTasks.addAll(toPartition.getInverse(partition));
	}
	
	public IStrategoTuple stopAnalysis(IStrategoString partition) {
		for(IStrategoInt removed : initialTasks)
			removeTask(removed, partition);
		
		return factory.makeTuple(factory.makeList(initialTasks), factory.makeList(addedTasks));
	}
	
	private void removeTask(IStrategoInt taskID, IStrategoString partition) {
		toPartition.remove(taskID, partition);
		if(!toPartition.containsKey(taskID)) {
			toInstruction.remove(taskID);
			toDependency.removeAll(taskID);
			toResult.remove(taskID);
		}
	}
}
<commit_msg>garbage collection<commit_after>package org.metaborg.runtime.task;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.spoofax.interpreter.terms.IStrategoInt;
import org.spoofax.interpreter.terms.IStrategoList;
import org.spoofax.interpreter.terms.IStrategoString;
import org.spoofax.interpreter.terms.IStrategoTerm;
import org.spoofax.interpreter.terms.IStrategoTuple;
import org.spoofax.interpreter.terms.ITermFactory;

import static com.google.common.collect.Sets.filter;

import static com.google.common.base.Predicates.in;
import static com.google.common.base.Predicates.not;

public class Tasks {
	
	private final ITermFactory factory;
	
	private final ManyToManyMap<IStrategoInt, IStrategoString> toPartition = ManyToManyMap.create();
	private final ManyToManyMap<IStrategoInt, IStrategoInt> toDependency   = ManyToManyMap.create();
	private final Map<IStrategoInt, IStrategoTerm> toInstruction = new ConcurrentHashMap<IStrategoInt, IStrategoTerm>();
	private final Map<IStrategoInt, IStrategoList> toResult      = new ConcurrentHashMap<IStrategoInt, IStrategoList>();
	
	private final Set<IStrategoInt> addedTasks = new HashSet<IStrategoInt>();
	private final Set<IStrategoInt> removedTasks = new HashSet<IStrategoInt>();
	
	private final Set<IStrategoInt> tasks    = toInstruction.keySet();
	private final Set<IStrategoInt> garbage  = filter(tasks, not(in(toPartition.keys())));
	private final Set<IStrategoInt> solved   = toResult.keySet();
	private final Set<IStrategoInt> stalled  = toDependency.keySet();
	// private final Set<IStrategoInt> solvable = filter(tasks, not(or(in(solved), in(stalled))));
	
	public Tasks(ITermFactory factory) {
		this.factory = factory;
	}
	
	public IStrategoInt addTask(IStrategoString partition, IStrategoList dependencies, IStrategoTerm instruction) {
		
		IStrategoInt taskID = factory.makeInt(instruction.hashCode());
		if(toInstruction.put(taskID, instruction) == null)
			addedTasks.add(taskID);
		removedTasks.remove(taskID);
		
		toPartition.put(taskID, partition);
		
        while(!dependencies.isEmpty()) {
        	toDependency.put(taskID, (IStrategoInt) dependencies.head());
            dependencies = dependencies.tail();
        }
        
        return taskID;
	}
	
	public void addResult(IStrategoInt taskID, IStrategoTerm result) {
		toResult.put(taskID, factory.makeListCons(result, factory.makeList()));
	}
	
	public void addResult(IStrategoInt taskID, IStrategoList resultList) {
		toResult.put(taskID, resultList);
	}
	
	public void startAnalysis(IStrategoString partition) {
		addedTasks.clear();
		removedTasks.clear();
		removedTasks.addAll(toPartition.getInverse(partition));
	}
	
	public IStrategoTuple stopAnalysis(IStrategoString partition) {

		for(IStrategoInt removed : removedTasks)
			toPartition.remove(removed, partition);
		
		collectGarbage();
		
		return factory.makeTuple(factory.makeList(addedTasks), factory.makeList(removedTasks));
	}
	
//	private void removeTask(IStrategoInt taskID, IStrategoString partition) {
//		toPartition.remove(taskID, partition);
//		
//		if(!toPartition.containsKey(taskID)) {
//			toDependency.removeAll(taskID);
//			toInstruction.remove(taskID);
//			toResult.remove(taskID);
//		}
//	}
	
	private void collectGarbage() {
		tasks.removeAll(garbage);
		solved.removeAll(garbage);
		stalled.removeAll(garbage);
	}
 }

{'fraction_non_alphanumeric': 0.10590305551111823, 'fraction_numerical': 0.0, 'mean_word_length': 5.46535677352637, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3436246', 'n_tokens_mistral': 2145, 'n_tokens_neox': 2029, 'n_words': 350}