<commit_before>package lbms.plugins.mldht.kad.tasks;

import lbms.plugins.mldht.kad.DHT;
import lbms.plugins.mldht.kad.DHTConstants;
import lbms.plugins.mldht.kad.Key;
import lbms.plugins.mldht.kad.Node;
import lbms.plugins.mldht.kad.RPCServer;
import lbms.plugins.mldht.kad.tasks.IterativeLookupCandidates.LookupGraphNode;
import lbms.plugins.mldht.kad.utils.AddressUtils;
import lbms.plugins.mldht.kad.utils.PopulationEstimator;
import lbms.plugins.mldht.kad.DHT.LogLevel;

import java.util.stream.Collectors;

public abstract class IteratingTask extends TargetedTask {
	
	ClosestSet closest;
	IterativeLookupCandidates todo;
	
	public IteratingTask(Key target, RPCServer srv, Node node) {
		super(target, srv, node);
		todo = new IterativeLookupCandidates(target);
		closest = new ClosestSet(target, DHTConstants.MAX_ENTRIES_PER_BUCKET);
	}
	
	@Override
	public int getTodoCount() {
		return (int) todo.cand().count();
	}
	
	public String closestDebug() {
		return this.closest.ids().<String>map(k -> {
			return k + "  " + targetKey.distance(k) + " " + PopulationEstimator.distanceToDouble(k, targetKey) + " src:" + todo.allCand().unordered().filter(e -> e.getKey().getID().equals(k)).findAny().get().getValue().sources.size();
		}).collect(Collectors.joining("\n"));
	}
	
	protected void logClosest() {
		Key farthest = closest.tail();
		
		DHT.log(this.toString() + "\n" +

				"Task "+ getTaskID() +"  done " + counts + " " + closest + "\n" + targetKey + "\n" + closestDebug()  + "\n" +
				

				todo.allCand().sorted(todo.comp()).filter(me -> {
					return targetKey.threeWayDistance(me.getKey().getID(), farthest) <= 0;
				}).<String>map(e -> {
					return e.getKey().getID() + " " + targetKey.distance(e.getKey().getID()) + " " + AddressUtils.toString(e.getKey().getAddress()) + " src:" + e.getValue().sources.size() + " call:" + todo.numCalls(e.getKey()) + " rsp:" + todo.numRsps(e.getKey()) + " " + e.getValue().sources.stream().map(LookupGraphNode::toKbe).collect(Collectors.toList());
				}).collect(Collectors.joining("\n"))

				, LogLevel.Verbose);

	}

}
<commit_msg>avoid recalculating data that's now recorded explicitly<commit_after>package lbms.plugins.mldht.kad.tasks;

import lbms.plugins.mldht.kad.DHT;
import lbms.plugins.mldht.kad.DHTConstants;
import lbms.plugins.mldht.kad.Key;
import lbms.plugins.mldht.kad.Node;
import lbms.plugins.mldht.kad.RPCServer;
import lbms.plugins.mldht.kad.RPCState;
import lbms.plugins.mldht.kad.tasks.IterativeLookupCandidates.LookupGraphNode;
import lbms.plugins.mldht.kad.utils.AddressUtils;
import lbms.plugins.mldht.kad.utils.PopulationEstimator;
import lbms.plugins.mldht.kad.DHT.LogLevel;

import java.util.stream.Collectors;

public abstract class IteratingTask extends TargetedTask {
	
	ClosestSet closest;
	IterativeLookupCandidates todo;
	
	public IteratingTask(Key target, RPCServer srv, Node node) {
		super(target, srv, node);
		todo = new IterativeLookupCandidates(target);
		closest = new ClosestSet(target, DHTConstants.MAX_ENTRIES_PER_BUCKET);
	}
	
	@Override
	public int getTodoCount() {
		return (int) todo.cand().count();
	}
	
	public String closestDebug() {
		return this.closest.ids().<String>map(k -> {
			return k + "  " + targetKey.distance(k) + " " + PopulationEstimator.distanceToDouble(k, targetKey) + " src:" + todo.allCand().unordered().filter(e -> e.getKey().getID().equals(k)).findAny().get().getValue().sources.size();
		}).collect(Collectors.joining("\n"));
	}
	
	protected void logClosest() {
		Key farthest = closest.tail();
		
		DHT.log(this.toString() + "\n" +

				"Task "+ getTaskID() +"  done " + counts + " " + closest + "\n" + targetKey + "\n" + closestDebug()  + "\n" +
				

				todo.allCand().sorted(todo.comp()).filter(me -> {
					return targetKey.threeWayDistance(me.getKey().getID(), farthest) <= 0;
				}).<String>map(e -> {
					LookupGraphNode node = e.getValue();
					return e.getKey().getID() + " " + targetKey.distance(e.getKey().getID()) + " " + AddressUtils.toString(e.getKey().getAddress()) + " src:" + node.sources.size() + " call:" + node.calls.size() + " rsp:" + node.calls.stream().filter(c -> c.state() == RPCState.RESPONDED).count() + " " + e.getValue().sources.stream().map(LookupGraphNode::toKbe).collect(Collectors.toList());
				}).collect(Collectors.joining("\n"))

				, LogLevel.Verbose);

	}

}

{'fraction_non_alphanumeric': 0.17551963048498845, 'fraction_numerical': 0.00046189376443418013, 'mean_word_length': 4.475347661188369, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3626669', 'n_tokens_mistral': 1660, 'n_tokens_neox': 1532, 'n_words': 227}