<commit_before>package eu.diachron.qualitymetrics.intrinsic.conciseness;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import org.apache.log4j.Logger;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.HTreeMap;

import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.sparql.core.Quad;

import de.unibonn.iai.eis.luzzu.assessment.ComplexQualityMetric;
import de.unibonn.iai.eis.luzzu.datatypes.ProblemList;
import eu.diachron.semantics.vocabulary.DQM;

/**
 * @author <NAME>
 * Provides a measure of the redundancy of the dataset at the data level, by calculating the 
 * Extensional Conciseness metric, which is part of the Conciseness dimension.
 */
public class ExtensionalConcisenessNew implements ComplexQualityMetric {
	
	transient private static Logger logger = Logger.getLogger(ExtensionalConciseness.class);
	
	transient private final Resource METRIC_URI = DQM.ExtensionalConcisenessMetric;
	
	/**
	 * MapDB database, used to persist the Map containing the instances found to be declared in the dataset
	 */
	transient private static DB mapDB = DBMaker.newTempFileDB()
			.closeOnJvmShutdown()
			.deleteFilesAfterClose()
        	.make();
	
	/**
	 * Map indexing the subjects detected during the computation of the metric. Every subject is identified 
	 * by a different id (URI), which serves as key of the map. The value of each subject consists of a 
	 * resource. A ConcurrentHashMap is used since it is synchronized and metric instances ought to be thread safe.
	 */
	 private HTreeMap<String, String> pMapSubjects = mapDB.createHashMap("extensional-conciseness-map").make(); //subject , "triples string"
	 
	 private int totalInstances = 0;

	 private boolean afterInvoked = false;
	
	
	/**
	 * Re-computes the value of the Extensional Conciseness Metric, by considering a new quad provided.
	 * @param quad The new quad to be considered in the computation of the metric. Must be not null.
	 */
	public void compute(Quad quad) {
		
		String triples = pMapSubjects.get(quad.getSubject().getURI());
		
		
		String value = quad.getPredicate().getURI().toString() + " " + quad.getObject().toString() + " ";
		if (triples == null) { 
			pMapSubjects.put(quad.getSubject().getURI(), value);
		} 
		else {
			String concat = triples + value;
			pMapSubjects.put(quad.getSubject().getURI(), concat);
		}
		
	}


	public void before(Object... args) {
		// Do Nothing
	}
	
	
	int nonUniqueInstances = 0;
	public void after(Object... args) {
		totalInstances = pMapSubjects.size();
		afterInvoked = true;
		
		
		Iterator<Map.Entry<String,String>> iter = pMapSubjects.entrySet().iterator();
		Set<String> duplicates = new HashSet<String>(); 
		while (iter.hasNext()) {
			Map.Entry<String, String> nxt = iter.next();
			String key = nxt.getKey();
			String value = nxt.getValue();
			if (duplicates.contains(key)) continue; //we do not check those metrics we already checked
			for(String otherKey : pMapSubjects.keySet()){
				if (key.equals(otherKey)) continue;
				if (value.equals(pMapSubjects.get(otherKey))) {
					nonUniqueInstances++;
					duplicates.add(otherKey);
					pMapSubjects.remove(otherKey);
				} 
			}
		}
	}
	
	
	/**
	 * Returns the current value of the Extensional Conciseness Metric, computed as the ratio of the 
	 * Number of Unique Subjects to the Total Number of Subjects. 
	 * Subjects are the objects being described by the quads provided on invocations to the compute 
	 * method, each subject is identified by its URI (the value of the subject attribute of the quad). 
	 * Uniqueness of subjects is determined from its properties: one subject is said to be unique 
	 * if and only if there is no other subject equivalent to it.
	 * - Note that two equivalent subjects may have different ids (URIs).
	 * @return Current value of the Extensional Conciseness Metric: (No. of Unique Subjects / Total No. of Subjects)
	 */<commit_msg>Improved algorithm<commit_after>package eu.diachron.qualitymetrics.intrinsic.conciseness;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.log4j.Logger;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.HTreeMap;

import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.sparql.core.Quad;

import de.unibonn.iai.eis.luzzu.assessment.ComplexQualityMetric;
import de.unibonn.iai.eis.luzzu.datatypes.ProblemList;
import eu.diachron.semantics.vocabulary.DQM;

/**
 * @author <NAME>
 * Provides a measure of the redundancy of the dataset at the data level, by calculating the 
 * Extensional Conciseness metric, which is part of the Conciseness dimension.
 */
public class ExtensionalConcisenessNew implements ComplexQualityMetric {
	
	transient private static Logger logger = Logger.getLogger(ExtensionalConciseness.class);
	
	transient private final Resource METRIC_URI = DQM.ExtensionalConcisenessMetric;
	
	/**
	 * MapDB database, used to persist the Map containing the instances found to be declared in the dataset
	 */
	transient private static DB mapDB = DBMaker.newTempFileDB()
			.closeOnJvmShutdown()
			.deleteFilesAfterClose()
        	.make();
	
	/**
	 * Map indexing the subjects detected during the computation of the metric. Every subject is identified 
	 * by a different id (URI), which serves as key of the map. The value of each subject consists of a 
	 * resource. A ConcurrentHashMap is used since it is synchronized and metric instances ought to be thread safe.
	 */
	 private HTreeMap<String, String> pMapSubjects = mapDB.createHashMap("extensional-conciseness-map").make(); //subject , "triples string"
	 
	 private long totalInstances = 0;

	 private boolean afterInvoked = false;
	
	
	/**
	 * Re-computes the value of the Extensional Conciseness Metric, by considering a new quad provided.
	 * @param quad The new quad to be considered in the computation of the metric. Must be not null.
	 */
	public void compute(Quad quad) {
		
		String triples = pMapSubjects.get(quad.getSubject().getURI());
		
		
		String value = quad.getPredicate().getURI().toString() + " " + quad.getObject().toString() + " ";
		if (triples == null) { 
			pMapSubjects.put(quad.getSubject().getURI(), value);
			totalInstances++;
		} 
		else {
			String concat = triples + value;
			pMapSubjects.put(quad.getSubject().getURI(), concat);
		}
		
	}


	public void before(Object... args) {
		// Do Nothing
	}
	
	
	long nonUniqueInstances = 0;
	public void after(Object... args) {
//		totalInstances = pMapSubjects.size();
		afterInvoked = true;
		
		 HTreeMap<String, String> pReversedMap = mapDB.createHashMap("extensional-conciseness-reversed-map").make(); //"triples string", Subject - the reverse of pMapSubjects
		 HTreeMap<String, List<String>> duplicates = mapDB.createHashMap("duplicates").make(); //  "triples string", Subject - stores duplicate instances
		 Set<String> values = mapDB.createHashSet("values").make();

		
		for(String instance : pMapSubjects.keySet()){
			String triples = pMapSubjects.get(instance);
			pReversedMap.put(triples, instance);
			if (!values.add(triples)) {
				if (duplicates.containsKey(triples)) {
					List<String> lst = duplicates.get(triples);
					lst.add(instance);
					duplicates.put(triples, lst);
				} else {
					List<String> lst = new ArrayList<String>();
					lst.add(instance);
					duplicates.put(triples, lst);
					nonUniqueInstances++;
				}
			}
		}
		
//		nonUniqueInstances = duplicates.size();
		
		duplicates.close();
	}
	
	
	/**
	 * Returns the current value of the Extensional Conciseness Metric, computed as the ratio of the 
	 * Number of Unique Subjects to the Total Number of Subjects. 
	 * Subjects are the objects being described by the quads provided on invocations to the compute 
	 * method, each subject is identified by its URI (the value of the subject attribute of the quad). 
	 * Uniqueness of subjects is determined from its properties: one subject is said to be unique 
	 * if and only if there is no other subject equivalent to it.
	 * - Note that two equivalent subjects may have different ids (URIs).
	 * @return Current value of the Extensional Conciseness Metric: (No. of Unique Subjects / Total No. of Subjects)
	 */
{'fraction_non_alphanumeric': 0.09946010797840432, 'fraction_numerical': 0.0007198560287942411, 'mean_word_length': 3.8977673325499413, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2658808', 'n_tokens_mistral': 2763, 'n_tokens_neox': 2433, 'n_words': 804}