<commit_before>package com.dataart.spreadsheetanalytics.engine.execgraph;

import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.CELL_WITH_FORMULA;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.CELL_WITH_VALUE;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.CONSTANT_VALUE;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.EMPTY_CELL;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.FUNCTION;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.IF;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.OPERATOR;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.RANGE;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.isCell;
import static java.lang.String.format;
import static java.lang.String.join;
import static java.util.Arrays.asList;
import static java.util.stream.Collectors.toList;
import static org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName.FORMULA_PTG;
import static org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName.NAME;
import static org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName.TYPE;
import static org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName.VALUE;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.poi.common.execgraph.IExecutionGraphBuilder;
import org.apache.poi.common.execgraph.IExecutionGraphVertex;
import org.apache.poi.common.execgraph.IExecutionGraphVertexProperty;
import org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName;
import org.apache.poi.ss.formula.eval.BlankEval;
import org.apache.poi.ss.formula.eval.ErrorEval;
import org.apache.poi.ss.formula.eval.ValueEval;
import org.apache.poi.ss.formula.functions.Area2DValues;
import org.apache.poi.ss.formula.ptg.AbstractFunctionPtg;
import org.apache.poi.ss.formula.ptg.AddPtg;
import org.apache.poi.ss.formula.ptg.AreaPtg;
import org.apache.poi.ss.formula.ptg.DividePtg;
import org.apache.poi.ss.formula.ptg.EqualPtg;
import org.apache.poi.ss.formula.ptg.GreaterThanPtg;
import org.apache.poi.ss.formula.ptg.LessThanPtg;
import org.apache.poi.ss.formula.ptg.MultiplyPtg;
import org.apache.poi.ss.formula.ptg.NameXPxg;
import org.apache.poi.ss.formula.ptg.NotEqualPtg;
import org.apache.poi.ss.formula.ptg.ParenthesisPtg;
import org.apache.poi.ss.formula.ptg.Ptg;
import org.apache.poi.ss.formula.ptg.Ref3DPxg;
import org.apache.poi.ss.formula.ptg.RefPtg;
import org.apache.poi.ss.formula.ptg.ScalarConstantPtg;
import org.apache.poi.ss.formula.ptg.SubtractPtg;
import org.apache.poi.ss.formula.ptg.UnaryPlusPtg;
import org.apache.poi.ss.formula.ptg.ValueOperatorPtg;
import org.jgrapht.DirectedGraph;
import org.jgrapht.graph.DefaultDirectedGraph;
import org.jgrapht.graph.DefaultEdge;

import com.dataart.spreadsheetanalytics.api.model.ICellAddress;
import com.dataart.spreadsheetanalytics.api.model.ICellValue;
import com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type;
import com.dataart.spreadsheetanalytics.model.CellAddress;
import com.dataart.spreadsheetanalytics.model.CellFormulaExpression;
import com.dataart.spreadsheetanalytics.model.CellValue;

/**
 * TODO: write about internal representation, not thread safe, one instance per
 * calculation, etc.
 *
 */
public class PoiExecutionGraphBuilder implements IExecutionGraphBuilder {

    protected static final String CONSTANT_VALUE_NAME = "VALUE";
    protected static final String UNDEFINED_EXTERNAL_FUNCTION = "#external#";
    protected static final Set<String> POI_VALUE_REDUNDANT_SYMBOLS = new HashSet<>(Arrays.asList("[", "]"));

	protected final DirectedGraph<IExecutionGraphVertex, DefaultEdge> dgraph;
	protected Map<ValueEval, IExecutionGraphVertex> valueToVertex;
	protected Map<String, Set<IExecutionGraphVertex>> addressToVertices;
	protected Set<com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex> connectedGraphVertices;

	public PoiExecutionGraphBuilder() {
		this.dgraph =  new DefaultDirectedGraph<>(DefaultEdge.class);
		this.valueToVertex = new HashMap<>();
		this.addressToVertices = new HashMap<>();
		connectedGraphVertices = new HashSet<>();
	}

	public ExecutionGraph get() {
	    ExecutionGraph result = ExecutionGraph.wrap(dgraph);
	    result.setVertices(connectedGraphVertices);
		return result;
	}

	public ExecutionGraph getSingleNodeGraph(ICellAddress address) {
		DirectedGraph<IExecutionGraphVertex, DefaultEdge> emptyGraph = new DefaultDirectedGraph<>(DefaultEdge.class);
		ExecutionGraphVertex vertex = new ExecutionGraphVertex(address.a1Address().address());
		vertex.property(TYPE).set(EMPTY_CELL);
		emptyGraph.addVertex(vertex);
		ExecutionGraph result = ExecutionGraph.wrap(emptyGraph);
		connectedGraphVertices.add(vertex);
		result.setVertices(connectedGraphVertices);
		return result;
	}

	/**
	 * This method should be used when creating a new vertex from a cell, so
	 * vertex name is a cell's address. New Vertex will be created any time this
	 * method is invoked. New vertex will be stored in
	 * address-to-set-of-vertices map.
	 */
	@Override
	public IExecutionGraphVertex createVertex(String address) {
		// create new vertex object
		ExecutionGraphVertex v = new ExecutionGraphVertex(address);

		// add vertex to actual graph
		dgraph.addVertex(v);

		// put new vertex to set of vertices with the same address, since they
		// all must have the same set of properties and values
		Set<IExecutionGraphVertex> vs = addressToVertices.containsKey(address) ? addressToVertices.get(address) : new HashSet<>();
		vs.add(v);
		addressToVertices.put(address, vs);

		return v;
	}

	@Override
	public IExecutionGraphVertex createVertex(Ptg ptg) {
		if (isSkipVertex(ptg)) { return null; }
		
		boolean isCell = ptg instanceof RefPtg;
		String name = ptgToString(ptg);

		if (isCell) { // cell
			return createVertex(name);
		} else { // operation
			ExecutionGraphVertex v = new ExecutionGraphVertex(name);
			dgraph.addVertex(v);
			return v;
		}
	}

	@Override
	public void connect(IExecutionGraphVertex from, IExecutionGraphVertex to) {
		dgraph.addEdge(from, to);
	}

	@Override
	public void removeVertex(IExecutionGraphVertex vertex) {
		if (vertex == null) { return; }
		dgraph.removeVertex(vertex);
	}

	@Override
	public void putVertexToCache(ValueEval value, IExecutionGraphVertex vertex) {
		if (value == null) { throw new IllegalArgumentException("ValueEval to assosiate vertex with cannot be null."); }
		valueToVertex.put(value, vertex);
	}

	@Override
	public IExecutionGraphVertex getVertexFromCache(ValueEval value) {
		if (value == null) { throw new IllegalArgumentException("ValueEval to assosiate vertex with cannot be null."); }
		return valueToVertex.get(value);
	}

	@Override
	public void putVertexToCache(String address, IExecutionGraphVertex vertex) {
		if (!addressToVertices.containsKey(address)) { addressToVertices.put(address, new HashSet<>()); }
		addressToVertices.get(address).add(vertex);
	}

	@Override
	public Set<IExecutionGraphVertex> getVerticesFromCache(String address) {
		return addressToVertices.get(address);
	}

	@Override
	public void putVertexToCache(int row, int column, IExecutionGraphVertex vertex) {
		putVertexToCache(CellAddress.toA1Address(row, column), vertex);
	}

	@Override
	public Set<IExecutionGraphVertex> getVerticesFromCache(int row, int column) {
		return getVerticesFromCache(CellAddress.toA1Address(row, column));
	}

	@Override<commit_msg>#64 saving arguments into cache in a prorer way<commit_after>package com.dataart.spreadsheetanalytics.engine.execgraph;

import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.CELL_WITH_FORMULA;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.CELL_WITH_VALUE;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.CONSTANT_VALUE;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.EMPTY_CELL;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.FUNCTION;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.IF;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.OPERATOR;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.RANGE;
import static com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type.isCell;
import static java.lang.String.format;
import static java.lang.String.join;
import static java.util.Arrays.asList;
import static java.util.stream.Collectors.toList;
import static org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName.FORMULA_PTG;
import static org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName.NAME;
import static org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName.TYPE;
import static org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName.VALUE;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.poi.common.execgraph.IExecutionGraphBuilder;
import org.apache.poi.common.execgraph.IExecutionGraphVertex;
import org.apache.poi.common.execgraph.IExecutionGraphVertexProperty;
import org.apache.poi.common.execgraph.IExecutionGraphVertexProperty.PropertyName;
import org.apache.poi.ss.formula.eval.BlankEval;
import org.apache.poi.ss.formula.eval.ErrorEval;
import org.apache.poi.ss.formula.eval.ValueEval;
import org.apache.poi.ss.formula.functions.Area2DValues;
import org.apache.poi.ss.formula.ptg.AbstractFunctionPtg;
import org.apache.poi.ss.formula.ptg.AddPtg;
import org.apache.poi.ss.formula.ptg.AreaPtg;
import org.apache.poi.ss.formula.ptg.DividePtg;
import org.apache.poi.ss.formula.ptg.EqualPtg;
import org.apache.poi.ss.formula.ptg.GreaterThanPtg;
import org.apache.poi.ss.formula.ptg.LessThanPtg;
import org.apache.poi.ss.formula.ptg.MultiplyPtg;
import org.apache.poi.ss.formula.ptg.NameXPxg;
import org.apache.poi.ss.formula.ptg.NotEqualPtg;
import org.apache.poi.ss.formula.ptg.ParenthesisPtg;
import org.apache.poi.ss.formula.ptg.Ptg;
import org.apache.poi.ss.formula.ptg.Ref3DPxg;
import org.apache.poi.ss.formula.ptg.RefPtg;
import org.apache.poi.ss.formula.ptg.ScalarConstantPtg;
import org.apache.poi.ss.formula.ptg.SubtractPtg;
import org.apache.poi.ss.formula.ptg.UnaryPlusPtg;
import org.apache.poi.ss.formula.ptg.ValueOperatorPtg;
import org.jgrapht.DirectedGraph;
import org.jgrapht.graph.DefaultDirectedGraph;
import org.jgrapht.graph.DefaultEdge;

import com.dataart.spreadsheetanalytics.api.model.ICellAddress;
import com.dataart.spreadsheetanalytics.api.model.ICellValue;
import com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex.Type;
import com.dataart.spreadsheetanalytics.model.CellAddress;
import com.dataart.spreadsheetanalytics.model.CellFormulaExpression;
import com.dataart.spreadsheetanalytics.model.CellValue;

/**
 * TODO: write about internal representation, not thread safe, one instance per
 * calculation, etc.
 *
 */
public class PoiExecutionGraphBuilder implements IExecutionGraphBuilder {

    protected static final String CONSTANT_VALUE_NAME = "VALUE";
    protected static final String UNDEFINED_EXTERNAL_FUNCTION = "#external#";
    protected static final Set<String> POI_VALUE_REDUNDANT_SYMBOLS = new HashSet<>(Arrays.asList("[", "]"));

	protected final DirectedGraph<IExecutionGraphVertex, DefaultEdge> dgraph;
	protected Map<ValueEval, Stack<IExecutionGraphVertex>> valueToVertex;
	protected Map<String, Set<IExecutionGraphVertex>> addressToVertices;
	protected Set<com.dataart.spreadsheetanalytics.api.model.IExecutionGraphVertex> connectedGraphVertices;

	public PoiExecutionGraphBuilder() {
		this.dgraph =  new DefaultDirectedGraph<>(DefaultEdge.class);
		this.valueToVertex = new HashMap<>();
		this.addressToVertices = new HashMap<>();
		connectedGraphVertices = new HashSet<>();
	}

	public ExecutionGraph get() {
	    ExecutionGraph result = ExecutionGraph.wrap(dgraph);
	    result.setVertices(connectedGraphVertices);
		return result;
	}

	public ExecutionGraph getSingleNodeGraph(ICellAddress address) {
		DirectedGraph<IExecutionGraphVertex, DefaultEdge> emptyGraph = new DefaultDirectedGraph<>(DefaultEdge.class);
		ExecutionGraphVertex vertex = new ExecutionGraphVertex(address.a1Address().address());
		vertex.property(TYPE).set(EMPTY_CELL);
		emptyGraph.addVertex(vertex);
		ExecutionGraph result = ExecutionGraph.wrap(emptyGraph);
		connectedGraphVertices.add(vertex);
		result.setVertices(connectedGraphVertices);
		return result;
	}

	/**
	 * This method should be used when creating a new vertex from a cell, so
	 * vertex name is a cell's address. New Vertex will be created any time this
	 * method is invoked. New vertex will be stored in
	 * address-to-set-of-vertices map.
	 */
	@Override
	public IExecutionGraphVertex createVertex(String address) {
		// create new vertex object
		ExecutionGraphVertex v = new ExecutionGraphVertex(address);

		// add vertex to actual graph
		dgraph.addVertex(v);

		// put new vertex to set of vertices with the same address, since they
		// all must have the same set of properties and values
		Set<IExecutionGraphVertex> vs = addressToVertices.containsKey(address) ? addressToVertices.get(address) : new HashSet<>();
		vs.add(v);
		addressToVertices.put(address, vs);

		return v;
	}

	@Override
	public IExecutionGraphVertex createVertex(Ptg ptg) {
		if (isSkipVertex(ptg)) { return null; }
		
		boolean isCell = ptg instanceof RefPtg;
		String name = ptgToString(ptg);

		if (isCell) { // cell
			return createVertex(name);
		} else { // operation
			ExecutionGraphVertex v = new ExecutionGraphVertex(name);
			dgraph.addVertex(v);
			return v;
		}
	}

	@Override
	public void connect(IExecutionGraphVertex from, IExecutionGraphVertex to) {
		dgraph.addEdge(from, to);
	}

	@Override
	public void removeVertex(IExecutionGraphVertex vertex) {
		if (vertex == null) { return; }
		dgraph.removeVertex(vertex);
	}

	@Override
	public void putVertexToCache(ValueEval value, IExecutionGraphVertex vertex) {
		if (value == null) { throw new IllegalArgumentException("ValueEval to assosiate vertex with cannot be null."); }
		if (!valueToVertex.keySet().contains(value)) { valueToVertex.put(value, new Stack<IExecutionGraphVertex>()); }
		valueToVertex.get(value).push(vertex);
	}

	@Override
	public IExecutionGraphVertex getVertexFromCache(ValueEval value) {
		if (value == null) { throw new IllegalArgumentException("ValueEval to assosiate vertex with cannot be null."); }
		return valueToVertex.get(value).pop();
	}

	@Override
	public void putVertexToCache(String address, IExecutionGraphVertex vertex) {
		if (!addressToVertices.containsKey(address)) { addressToVertices.put(address, new HashSet<>()); }
		addressToVertices.get(address).add(vertex);
	}

	@Override
	public Set<IExecutionGraphVertex> getVerticesFromCache(String address) {
		return addressToVertices.get(address);
	}

	@Override
	public void putVertexToCache(int row, int column, IExecutionGraphVertex vertex) {
		putVertexToCache(CellAddress.toA1Address(row, column), vertex);
	}

	@Override
	public Set<IExecutionGraphVertex> getVerticesFromCache(int row, int column) {
		return getVerticesFromCache(CellAddress.toA1Address(row, column));
	}

	@Override
{'fraction_non_alphanumeric': 0.1008072085601652, 'fraction_numerical': 0.0007508916838746011, 'mean_word_length': 6.0096491228070175, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2283561', 'n_tokens_mistral': 5151, 'n_tokens_neox': 4924, 'n_words': 949}