<commit_before>package org.vitrivr.cineast.core.db;

import com.google.common.util.concurrent.ListenableFuture;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.adam.grpc.AdamGrpc;
import org.vitrivr.adam.grpc.AdamGrpc.*;
import org.vitrivr.adam.grpc.AdamGrpc.AckMessage.Code;
import org.vitrivr.adam.grpc.AdamGrpc.BooleanQueryMessage.WhereMessage;
import org.vitrivr.adam.grpc.AdamGrpc.DistanceMessage.DistanceType;
import org.vitrivr.cineast.core.config.QueryConfig;
import org.vitrivr.cineast.core.config.QueryConfig.Distance;
import org.vitrivr.cineast.core.data.StringDoublePair;
import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider;
import org.vitrivr.cineast.core.util.LogHelper;

import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;

public class ADAMproSelector implements DBSelector {

	private ADAMproWrapper adampro = new ADAMproWrapper();
	
	private FromMessage.Builder fromBuilder = FromMessage.newBuilder();
	private QueryMessage.Builder qmBuilder = QueryMessage.newBuilder();
	private NearestNeighbourQueryMessage.Builder nnqmBuilder = NearestNeighbourQueryMessage.newBuilder();
	private BooleanQueryMessage.Builder bqmBuilder = BooleanQueryMessage.newBuilder();
	private WhereMessage.Builder wmBuilder = WhereMessage.newBuilder();
	private DistanceMessage.Builder dmBuilder = DistanceMessage.newBuilder();
	private static final Logger LOGGER = LogManager.getLogger();

	private static ArrayList<String> hints = new ArrayList<>(1);
	private static ProjectionMessage projectionMessage;
	
	private static final DistanceMessage chisquared, correlation, cosine, hamming, jaccard, kullbackleibler, chebyshev, euclidean, squaredeuclidean, manhattan, spannorm;
	
	static{
		
		hints.add("exact");
		
		projectionMessage = AdamGrpc.ProjectionMessage.newBuilder().setAttributes(
				AdamGrpc.ProjectionMessage.AttributeNameMessage.newBuilder().addAttribute("ap_distance").addAttribute("id")).build();
		
		DistanceMessage.Builder dmBuilder = DistanceMessage.newBuilder();
		
		chisquared = dmBuilder.clear().setDistancetype(DistanceType.chisquared).build();
		correlation = dmBuilder.clear().setDistancetype(DistanceType.correlation).build();
		cosine = dmBuilder.clear().setDistancetype(DistanceType.cosine).build();
		hamming = dmBuilder.clear().setDistancetype(DistanceType.hamming).build();
		jaccard = dmBuilder.clear().setDistancetype(DistanceType.jaccard).build();
		kullbackleibler = dmBuilder.clear().setDistancetype(DistanceType.kullbackleibler).build();
		chebyshev = dmBuilder.clear().setDistancetype(DistanceType.chebyshev).build();
		euclidean = dmBuilder.clear().setDistancetype(DistanceType.euclidean).build();
		squaredeuclidean = dmBuilder.clear().setDistancetype(DistanceType.squaredeuclidean).build();
		manhattan = dmBuilder.clear().setDistancetype(DistanceType.manhattan).build();
		spannorm = dmBuilder.clear().setDistancetype(DistanceType.spannorm).build();
		
	}
	
	@Override
	public boolean open(String name) {
		this.fromBuilder.setEntity(name);
		return true;
	}

	@Override
	public boolean close() {
		this.adampro.close();
		return false;
	}

	private QueryMessage buildQueryMessage(ArrayList<String> hints, BooleanQueryMessage bqMessage, ProjectionMessage pMessage, NearestNeighbourQueryMessage nnqMessage){
		synchronized (qmBuilder) {
			qmBuilder.clear();
			qmBuilder.setFrom(fromBuilder);
			if(hints != null && !hints.isEmpty()){
				qmBuilder.addAllHints(hints);
			}
			if(bqMessage != null){
				qmBuilder.setBq(bqMessage);
			}
			if(pMessage != null){
				qmBuilder.setProjection(pMessage);
			}
			if(nnqMessage != null){
				qmBuilder.setNnq(nnqMessage);
			}
			
			return qmBuilder.build();
		}
	}
	
	private BooleanQueryMessage buildBooleanQueryMessage(WhereMessage where, WhereMessage...whereMessages){
		ArrayList<WhereMessage> tmp = new ArrayList<>(1 + (whereMessages == null ? 0 : whereMessages.length));
		tmp.add(where);
		if(whereMessages != null){
			for(WhereMessage w : whereMessages){
				tmp.add(w);
			}
		}
		synchronized (bqmBuilder) {
			bqmBuilder.clear();
			return bqmBuilder.addAllWhere(tmp).build();
		}
	}
	
	private WhereMessage buildWhereMessage(String key, String value){
		synchronized (wmBuilder) {
			wmBuilder.clear();
			return wmBuilder.setAttribute(key).setValue(value).build();
		}
	}

	private WhereMessage buildWhereMessage(String key, String... values){
		synchronized (wmBuilder) {
			wmBuilder.clear();
			StringBuilder sb = new StringBuilder();
			sb.append("IN ('");
			sb.append(String.join("', '", values));
			sb.append("')");
			LOGGER.debug("Querying... {}", sb.toString());
			return wmBuilder.setAttribute(key).setValue(sb.toString()).build();
		}
	}

	private NearestNeighbourQueryMessage buildNearestNeighbourQueryMessage(String column, FeatureVectorMessage fvm, int k, QueryConfig qc){
		synchronized (nnqmBuilder) {
			this.nnqmBuilder.clear();
			nnqmBuilder.setAttribute(column).setQuery(fvm).setK(k);<commit_msg>Removing Debug-logging from Wheremessage since it is too large<commit_after>package org.vitrivr.cineast.core.db;

import com.google.common.util.concurrent.ListenableFuture;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.adam.grpc.AdamGrpc;
import org.vitrivr.adam.grpc.AdamGrpc.*;
import org.vitrivr.adam.grpc.AdamGrpc.AckMessage.Code;
import org.vitrivr.adam.grpc.AdamGrpc.BooleanQueryMessage.WhereMessage;
import org.vitrivr.adam.grpc.AdamGrpc.DistanceMessage.DistanceType;
import org.vitrivr.cineast.core.config.QueryConfig;
import org.vitrivr.cineast.core.config.QueryConfig.Distance;
import org.vitrivr.cineast.core.data.StringDoublePair;
import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider;
import org.vitrivr.cineast.core.util.LogHelper;

import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;

public class ADAMproSelector implements DBSelector {

	private ADAMproWrapper adampro = new ADAMproWrapper();
	
	private FromMessage.Builder fromBuilder = FromMessage.newBuilder();
	private QueryMessage.Builder qmBuilder = QueryMessage.newBuilder();
	private NearestNeighbourQueryMessage.Builder nnqmBuilder = NearestNeighbourQueryMessage.newBuilder();
	private BooleanQueryMessage.Builder bqmBuilder = BooleanQueryMessage.newBuilder();
	private WhereMessage.Builder wmBuilder = WhereMessage.newBuilder();
	private DistanceMessage.Builder dmBuilder = DistanceMessage.newBuilder();
	private static final Logger LOGGER = LogManager.getLogger();

	private static ArrayList<String> hints = new ArrayList<>(1);
	private static ProjectionMessage projectionMessage;
	
	private static final DistanceMessage chisquared, correlation, cosine, hamming, jaccard, kullbackleibler, chebyshev, euclidean, squaredeuclidean, manhattan, spannorm;
	
	static{
		
		hints.add("exact");
		
		projectionMessage = AdamGrpc.ProjectionMessage.newBuilder().setAttributes(
				AdamGrpc.ProjectionMessage.AttributeNameMessage.newBuilder().addAttribute("ap_distance").addAttribute("id")).build();
		
		DistanceMessage.Builder dmBuilder = DistanceMessage.newBuilder();
		
		chisquared = dmBuilder.clear().setDistancetype(DistanceType.chisquared).build();
		correlation = dmBuilder.clear().setDistancetype(DistanceType.correlation).build();
		cosine = dmBuilder.clear().setDistancetype(DistanceType.cosine).build();
		hamming = dmBuilder.clear().setDistancetype(DistanceType.hamming).build();
		jaccard = dmBuilder.clear().setDistancetype(DistanceType.jaccard).build();
		kullbackleibler = dmBuilder.clear().setDistancetype(DistanceType.kullbackleibler).build();
		chebyshev = dmBuilder.clear().setDistancetype(DistanceType.chebyshev).build();
		euclidean = dmBuilder.clear().setDistancetype(DistanceType.euclidean).build();
		squaredeuclidean = dmBuilder.clear().setDistancetype(DistanceType.squaredeuclidean).build();
		manhattan = dmBuilder.clear().setDistancetype(DistanceType.manhattan).build();
		spannorm = dmBuilder.clear().setDistancetype(DistanceType.spannorm).build();
		
	}
	
	@Override
	public boolean open(String name) {
		this.fromBuilder.setEntity(name);
		return true;
	}

	@Override
	public boolean close() {
		this.adampro.close();
		return false;
	}

	private QueryMessage buildQueryMessage(ArrayList<String> hints, BooleanQueryMessage bqMessage, ProjectionMessage pMessage, NearestNeighbourQueryMessage nnqMessage){
		synchronized (qmBuilder) {
			qmBuilder.clear();
			qmBuilder.setFrom(fromBuilder);
			if(hints != null && !hints.isEmpty()){
				qmBuilder.addAllHints(hints);
			}
			if(bqMessage != null){
				qmBuilder.setBq(bqMessage);
			}
			if(pMessage != null){
				qmBuilder.setProjection(pMessage);
			}
			if(nnqMessage != null){
				qmBuilder.setNnq(nnqMessage);
			}
			
			return qmBuilder.build();
		}
	}
	
	private BooleanQueryMessage buildBooleanQueryMessage(WhereMessage where, WhereMessage...whereMessages){
		ArrayList<WhereMessage> tmp = new ArrayList<>(1 + (whereMessages == null ? 0 : whereMessages.length));
		tmp.add(where);
		if(whereMessages != null){
			for(WhereMessage w : whereMessages){
				tmp.add(w);
			}
		}
		synchronized (bqmBuilder) {
			bqmBuilder.clear();
			return bqmBuilder.addAllWhere(tmp).build();
		}
	}
	
	private WhereMessage buildWhereMessage(String key, String value){
		synchronized (wmBuilder) {
			wmBuilder.clear();
			return wmBuilder.setAttribute(key).setValue(value).build();
		}
	}

	private WhereMessage buildWhereMessage(String key, String... values){
		synchronized (wmBuilder) {
			wmBuilder.clear();
			StringBuilder sb = new StringBuilder();
			sb.append("IN ('");
			sb.append(String.join("', '", values));
			sb.append("')");
			return wmBuilder.setAttribute(key).setValue(sb.toString()).build();
		}
	}

	private NearestNeighbourQueryMessage buildNearestNeighbourQueryMessage(String column, FeatureVectorMessage fvm, int k, QueryConfig qc){
		synchronized (nnqmBuilder) {
			this.nnqmBuilder.clear();
			nnqmBuilder.setAttribute(column).setQuery(fvm).setK(k);
{'fraction_non_alphanumeric': 0.12058212058212059, 'fraction_numerical': 0.00099000099000099, 'mean_word_length': 5.886162235855488, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '974860', 'n_tokens_mistral': 3492, 'n_tokens_neox': 3079, 'n_words': 493}