<commit_before>package org.openlca.core.matrix.product.index;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.openlca.core.matrix.CalcExchange;
import org.openlca.core.matrix.LongPair;
import org.openlca.core.matrix.TechIndex;
import org.openlca.core.matrix.cache.MatrixCache;
import org.openlca.core.model.FlowType;
import org.openlca.core.model.ProcessLink;
import org.openlca.core.model.ProcessType;
import org.openlca.core.model.ProductSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TechIndexCutoffBuilder implements ITechIndexBuilder {

	private final Logger log = LoggerFactory.getLogger(getClass());

	private final MatrixCache cache;
	private final ProviderSearch providers;
	private final ProductSystem system;
	private final double cutoff;

	public TechIndexCutoffBuilder(MatrixCache cache, ProductSystem system, double cutoff) {
		this.cache = cache;
		this.cutoff = cutoff;
		this.system = system;
		this.providers = new ProviderSearch(cache.getProcessTable());
	}

	@Override
	public void setPreferredType(ProcessType type) {
		providers.setPreferredType(type);
	}

	@Override
	public TechIndex build(LongPair refProduct) {
		return build(refProduct, 1.0);
	}

	@Override
	public TechIndex build(LongPair refProduct, double demand) {
		log.trace("build product index for {} with cutoff=", refProduct,
				cutoff);
		TechIndex index = new TechIndex(refProduct);
		index.setDemand(demand);
		addSystemLinks(index);
		Graph g = new Graph(refProduct, demand);
		while (!g.next.isEmpty())
			g.handleNext();
		fillIndex(g, index);
		log.trace("created the index with {} products", index.size());
		return index;
	}

	private void addSystemLinks(TechIndex index) {
		if (system == null)
			return;
		for (ProcessLink link : system.getProcessLinks()) {
			LongPair provider = new LongPair(link.providerId, link.flowId);
			LongPair exchange = new LongPair(link.processId, link.exchangeId);
			index.putLink(exchange, provider);
		}
	}

	private void fillIndex(Graph g, TechIndex index) {
		for (Node node : g.nodes.values()) {
			if (node.state != NodeState.FOLLOWED)
				continue;
			for (Link link : node.inputLinks) {
				if (link.demand < cutoff)
					continue;
				Node provider = link.provider;
				LongPair exchange = LongPair.of(node.product.getFirst(),
						link.exchangeId);
				index.putLink(exchange, provider.product);
			}
		}
	}

	private class Graph {

		Node root;

		List<Node> next = new ArrayList<>();
		HashMap<LongPair, Node> nodes = new HashMap<>();

		Graph(LongPair refProduct, double demand) {
			this.root = new Node(refProduct, demand);
			root.product = refProduct;
			root.state = NodeState.WAITING;
			nodes.put(refProduct, root);
			next.add(root);
		}

		void handleNext() {

			log.trace("handle next layer with {} product nodes", next.size());

			// to minimize the re-scale calls we first sort the nodes by their
			// demands (see the compareTo method in Node)
			Collections.sort(next);

			Map<Long, List<CalcExchange>> nextExchanges = fetchNextExchanges();
			List<Node> nextLayer = new ArrayList<>();
			for (Node node : next) {
				node.state = NodeState.PROGRESS;
				List<CalcExchange> exchanges = nextExchanges.get(
						node.product.getFirst());
				CalcExchange output = getOutput(node, exchanges);
				if (output == null)
					continue;
				node.outputAmount = amount(output);
				node.scalingFactor = node.demand / node.outputAmount;
				followInputs(node, exchanges, nextLayer);
				node.state = NodeState.FOLLOWED;
			}
			next.clear();
			next.addAll(nextLayer);
		}

		private void followInputs(Node node, List<CalcExchange> exchanges,
				List<Node> nextLayer) {
			for (CalcExchange linkExchange : providers.getLinkCandidates(exchanges)) {
				LongPair provider = providers.find(linkExchange);
				if (provider == null)
					continue;
				double inputAmount = amount(linkExchange);
				double inputDemand = node.scalingFactor * inputAmount;
				Node providerNode = nodes.get(provider);
				if (providerNode != null)
					checkSubGraph(inputDemand, providerNode, nextLayer, false);
				else {
					providerNode = createNode(inputDemand, provider,
							nextLayer);
				}
				Link link = new Link(providerNode, linkExchange.exchangeId,
						inputAmount, inputDemand);
				node.inputLinks.add(link);
			}
		}

		private Node createNode(double inputDemand, LongPair product,
				List<Node> nextLayer) {
			Node node = new Node(product, inputDemand);
			nodes.put(product, node);
			if (inputDemand < cutoff)
				node.state = NodeState.EXCLUDED;
			else {
				node.state = NodeState.WAITING;
				nextLayer.add(node);<commit_msg>waste: better names<commit_after>package org.openlca.core.matrix.product.index;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.openlca.core.matrix.CalcExchange;
import org.openlca.core.matrix.LongPair;
import org.openlca.core.matrix.TechIndex;
import org.openlca.core.matrix.cache.MatrixCache;
import org.openlca.core.model.FlowType;
import org.openlca.core.model.ProcessLink;
import org.openlca.core.model.ProcessType;
import org.openlca.core.model.ProductSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TechIndexCutoffBuilder implements ITechIndexBuilder {

	private final Logger log = LoggerFactory.getLogger(getClass());

	private final MatrixCache cache;
	private final ProviderSearch providers;
	private final ProductSystem system;
	private final double cutoff;

	public TechIndexCutoffBuilder(MatrixCache cache, ProductSystem system, double cutoff) {
		this.cache = cache;
		this.cutoff = cutoff;
		this.system = system;
		this.providers = new ProviderSearch(cache.getProcessTable());
	}

	@Override
	public void setPreferredType(ProcessType type) {
		providers.setPreferredType(type);
	}

	@Override
	public TechIndex build(LongPair refProduct) {
		return build(refProduct, 1.0);
	}

	@Override
	public TechIndex build(LongPair refProduct, double demand) {
		log.trace("build product index for {} with cutoff=", refProduct,
				cutoff);
		TechIndex index = new TechIndex(refProduct);
		index.setDemand(demand);
		addSystemLinks(index);
		Graph g = new Graph(refProduct, demand);
		while (!g.next.isEmpty())
			g.handleNext();
		fillIndex(g, index);
		log.trace("created the index with {} products", index.size());
		return index;
	}

	private void addSystemLinks(TechIndex index) {
		if (system == null)
			return;
		for (ProcessLink link : system.getProcessLinks()) {
			LongPair provider = new LongPair(link.providerId, link.flowId);
			LongPair exchange = new LongPair(link.processId, link.exchangeId);
			index.putLink(exchange, provider);
		}
	}

	private void fillIndex(Graph g, TechIndex index) {
		for (Node node : g.nodes.values()) {
			if (node.state != NodeState.FOLLOWED)
				continue;
			for (Link link : node.inputLinks) {
				if (link.demand < cutoff)
					continue;
				Node provider = link.provider;
				LongPair exchange = LongPair.of(node.product.getFirst(),
						link.exchangeId);
				index.putLink(exchange, provider.product);
			}
		}
	}

	private class Graph {

		Node root;

		List<Node> next = new ArrayList<>();
		HashMap<LongPair, Node> nodes = new HashMap<>();

		Graph(LongPair refProduct, double demand) {
			this.root = new Node(refProduct, demand);
			root.product = refProduct;
			root.state = NodeState.WAITING;
			nodes.put(refProduct, root);
			next.add(root);
		}

		void handleNext() {

			log.trace("handle next layer with {} product nodes", next.size());

			// to minimize the re-scale calls we first sort the nodes by their
			// demands (see the compareTo method in Node)
			Collections.sort(next);

			Map<Long, List<CalcExchange>> nextExchanges = fetchNextExchanges();
			List<Node> nextLayer = new ArrayList<>();
			for (Node node : next) {
				node.state = NodeState.PROGRESS;
				List<CalcExchange> exchanges = nextExchanges.get(
						node.product.getFirst());
				CalcExchange output = getOutput(node, exchanges);
				if (output == null)
					continue;
				node.outputAmount = amount(output);
				node.scalingFactor = node.demand / node.outputAmount;
				followLinks(node, exchanges, nextLayer);
				node.state = NodeState.FOLLOWED;
			}
			next.clear();
			next.addAll(nextLayer);
		}

		private void followLinks(Node node, List<CalcExchange> exchanges,
				List<Node> nextLayer) {
			for (CalcExchange linkExchange : providers.getLinkCandidates(exchanges)) {
				LongPair provider = providers.find(linkExchange);
				if (provider == null)
					continue;
				double amount = amount(linkExchange);
				double demand = node.scalingFactor * amount;
				Node providerNode = nodes.get(provider);
				if (providerNode != null)
					checkSubGraph(demand, providerNode, nextLayer, false);
				else {
					providerNode = createNode(demand, provider,
							nextLayer);
				}
				Link link = new Link(providerNode, linkExchange.exchangeId,
						amount, demand);
				node.inputLinks.add(link);
			}
		}

		private Node createNode(double inputDemand, LongPair product,
				List<Node> nextLayer) {
			Node node = new Node(product, inputDemand);
			nodes.put(product, node);
			if (inputDemand < cutoff)
				node.state = NodeState.EXCLUDED;
			else {
				node.state = NodeState.WAITING;
				nextLayer.add(node);
{'fraction_non_alphanumeric': 0.11237493417588204, 'fraction_numerical': 0.0008425487098472881, 'mean_word_length': 4.1191374663072775, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4744851', 'n_tokens_mistral': 3426, 'n_tokens_neox': 2885, 'n_words': 711}