<commit_before>package owltools.mooncat;

import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.log4j.Logger;
import org.semanticweb.owlapi.model.AxiomType;
import org.semanticweb.owlapi.model.IRI;
import org.semanticweb.owlapi.model.OWLAnnotation;
import org.semanticweb.owlapi.model.OWLAnnotationAssertionAxiom;
import org.semanticweb.owlapi.model.OWLAnnotationProperty;
import org.semanticweb.owlapi.model.OWLAnnotationValue;
import org.semanticweb.owlapi.model.OWLAxiom;
import org.semanticweb.owlapi.model.OWLClass;
import org.semanticweb.owlapi.model.OWLClassExpression;
import org.semanticweb.owlapi.model.OWLDataFactory;
import org.semanticweb.owlapi.model.OWLEntity;
import org.semanticweb.owlapi.model.OWLEquivalentClassesAxiom;
import org.semanticweb.owlapi.model.OWLLiteral;
import org.semanticweb.owlapi.model.OWLObjectIntersectionOf;
import org.semanticweb.owlapi.model.OWLObjectProperty;
import org.semanticweb.owlapi.model.OWLObjectSomeValuesFrom;
import org.semanticweb.owlapi.model.OWLOntology;
import org.semanticweb.owlapi.model.OWLOntologyChange;
import org.semanticweb.owlapi.model.OWLOntologyManager;
import org.semanticweb.owlapi.model.OWLSubClassOfAxiom;
import org.semanticweb.owlapi.reasoner.Node;
import org.semanticweb.owlapi.reasoner.OWLReasoner;
import org.semanticweb.owlapi.util.OWLEntityRenamer;

import owltools.graph.OWLGraphWrapper;

/**
 * 
 * @author cjm
 * 
 */
public class EquivalenceSetMergeUtil {

	private Logger LOG = Logger.getLogger(EquivalenceSetMergeUtil.class);

	OWLGraphWrapper graph;

	// map from species-class to generic-class
	OWLOntology ont;
	public OWLReasoner reasoner;
	Map<String,Double> prefixScoreMap = new HashMap<String,Double>();
	Map<OWLAnnotationProperty,Map<String,Double>> propertyPrefixScoreMap = 
			new HashMap<OWLAnnotationProperty,Map<String,Double>>();
	boolean isAddEquivalenceAxioms = true;


	public EquivalenceSetMergeUtil(OWLGraphWrapper g, OWLReasoner r) {
		graph = g;
		ont = graph.getSourceOntology();
		reasoner = r;
	}
	
	public void setPropertyPrefixScore(OWLAnnotationProperty p, String prefix, Double score) {
		LOG.info("Setting "+p+" priority: "+prefix+" = "+score);
		if (!propertyPrefixScoreMap.containsKey(p))
			propertyPrefixScoreMap.put(p, new HashMap<String,Double>());
		setPrefixScore(propertyPrefixScoreMap.get(p), prefix, score);
	}

	public void setPrefixScore(Map<String, Double> pmap, String prefix,
			Double score) {
		pmap.put(prefix, score);		
	}
	public void setPrefixScore(String prefix,
			Double score) {
		prefixScoreMap.put(prefix, score);		
	}

	/**
	 * 
	 */
	public void merge() {

		Set<Node<OWLClass>> nodes = new HashSet<Node<OWLClass>>();
		Map<OWLClass, Node<OWLClass>> nodeByRep = 
				new HashMap<OWLClass, Node<OWLClass>>();
		for (OWLClass c : ont.getClassesInSignature()) {
			Node<OWLClass> n = reasoner.getEquivalentClasses(c);
			if (n.getSize() > 1) {
				nodes.add(n);
				nodeByRep.put(c, n);
			}			
		}
		LOG.info("TOTAL SETS-OF-SETS (redundant): "+nodeByRep.keySet().size());

		Map<OWLEntity,IRI> e2iri = new HashMap<OWLEntity,IRI>();

		Set<OWLClass> seenClasses = new HashSet<OWLClass>();
		Set <OWLAxiom> newAxioms = new HashSet<OWLAxiom>();
		
		for (Node<OWLClass> n : nodes) {
			boolean isSeen = false;
			for (OWLClass c : n.getEntities()) {
				if (seenClasses.contains(c)) {
					isSeen = true;
					break;
				}
				seenClasses.add(c);
			}
			if (isSeen) {
				continue;
			}
			if (true) {
				OWLClass rep = null;
				Double best = null;
				for (OWLClass c : n.getEntities()) {
					Double score = getScore(c, prefixScoreMap);
					if (best == null || (score != null && score > best)) {
						rep = c;
						best = score;
					}
				}
				for (OWLClass c : n.getEntities()) {
					if (c.equals(rep))
						continue;
					LOG.info(c + " --> "+rep);
					e2iri.put(c, rep.getIRI());
					if (isAddEquivalenceAxioms) {
						// note: this creates a dangler
						newAxioms.add(graph.getDataFactory().getOWLEquivalentClassesAxiom(c, rep));
					}
				}
			}

			// some properties may be desired to have cardinality = 1
			for (OWLAnnotationProperty p : propertyPrefixScoreMap.keySet()) {
				Map<String, Double> pmap = propertyPrefixScoreMap.get(p);
				OWLClass rep = null;
				Double best = null;
				for (OWLClass c : n.getEntities()) {
					String v = graph.getAnnotationValue(c, p);
					if (v == null || v.equals(""))
						continue;
					LOG.info(c + " . "+p+" = "+v);
					Double score = getScore(c, pmap);
					if (best == null || (score != null && score > best)) {<commit_msg>keep in ECAa as xrefs<commit_after>package owltools.mooncat;

import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.log4j.Logger;
import org.obolibrary.oboformat.parser.OBOFormatConstants.OboFormatTag;
import org.semanticweb.owlapi.model.AxiomType;
import org.semanticweb.owlapi.model.IRI;
import org.semanticweb.owlapi.model.OWLAnnotation;
import org.semanticweb.owlapi.model.OWLAnnotationAssertionAxiom;
import org.semanticweb.owlapi.model.OWLAnnotationProperty;
import org.semanticweb.owlapi.model.OWLAnnotationValue;
import org.semanticweb.owlapi.model.OWLAxiom;
import org.semanticweb.owlapi.model.OWLClass;
import org.semanticweb.owlapi.model.OWLClassExpression;
import org.semanticweb.owlapi.model.OWLDataFactory;
import org.semanticweb.owlapi.model.OWLEntity;
import org.semanticweb.owlapi.model.OWLEquivalentClassesAxiom;
import org.semanticweb.owlapi.model.OWLLiteral;
import org.semanticweb.owlapi.model.OWLObjectIntersectionOf;
import org.semanticweb.owlapi.model.OWLObjectProperty;
import org.semanticweb.owlapi.model.OWLObjectSomeValuesFrom;
import org.semanticweb.owlapi.model.OWLOntology;
import org.semanticweb.owlapi.model.OWLOntologyChange;
import org.semanticweb.owlapi.model.OWLOntologyManager;
import org.semanticweb.owlapi.model.OWLSubClassOfAxiom;
import org.semanticweb.owlapi.reasoner.Node;
import org.semanticweb.owlapi.reasoner.OWLReasoner;
import org.semanticweb.owlapi.util.OWLEntityRenamer;

import owltools.graph.OWLGraphWrapper;

/**
 * 
 * @author cjm
 * 
 */
public class EquivalenceSetMergeUtil {

	private Logger LOG = Logger.getLogger(EquivalenceSetMergeUtil.class);

	OWLGraphWrapper graph;

	// map from species-class to generic-class
	OWLOntology ont;
	public OWLReasoner reasoner;
	Map<String,Double> prefixScoreMap = new HashMap<String,Double>();
	Map<OWLAnnotationProperty,Map<String,Double>> propertyPrefixScoreMap = 
			new HashMap<OWLAnnotationProperty,Map<String,Double>>();
	boolean isAddEquivalenceAxioms = true;


	public EquivalenceSetMergeUtil(OWLGraphWrapper g, OWLReasoner r) {
		graph = g;
		ont = graph.getSourceOntology();
		reasoner = r;
	}
	
	public void setPropertyPrefixScore(OWLAnnotationProperty p, String prefix, Double score) {
		LOG.info("Setting "+p+" priority: "+prefix+" = "+score);
		if (!propertyPrefixScoreMap.containsKey(p))
			propertyPrefixScoreMap.put(p, new HashMap<String,Double>());
		setPrefixScore(propertyPrefixScoreMap.get(p), prefix, score);
	}

	public void setPrefixScore(Map<String, Double> pmap, String prefix,
			Double score) {
		pmap.put(prefix, score);		
	}
	public void setPrefixScore(String prefix,
			Double score) {
		prefixScoreMap.put(prefix, score);		
	}

	/**
	 * 
	 */
	public void merge() {

		Set<Node<OWLClass>> nodes = new HashSet<Node<OWLClass>>();
		Map<OWLClass, Node<OWLClass>> nodeByRep = 
				new HashMap<OWLClass, Node<OWLClass>>();
		for (OWLClass c : ont.getClassesInSignature()) {
			Node<OWLClass> n = reasoner.getEquivalentClasses(c);
			if (n.getSize() > 1) {
				nodes.add(n);
				nodeByRep.put(c, n);
			}			
		}
		LOG.info("TOTAL SETS-OF-SETS (redundant): "+nodeByRep.keySet().size());

		Map<OWLEntity,IRI> e2iri = new HashMap<OWLEntity,IRI>();

		Set<OWLClass> seenClasses = new HashSet<OWLClass>();
		Set <OWLAxiom> newAxioms = new HashSet<OWLAxiom>();
		
		for (Node<OWLClass> n : nodes) {
			boolean isSeen = false;
			for (OWLClass c : n.getEntities()) {
				if (seenClasses.contains(c)) {
					isSeen = true;
					break;
				}
				seenClasses.add(c);
			}
			if (isSeen) {
				continue;
			}
			if (true) {
				OWLClass rep = null;
				Double best = null;
				for (OWLClass c : n.getEntities()) {
					Double score = getScore(c, prefixScoreMap);
					if (best == null || (score != null && score > best)) {
						rep = c;
						best = score;
					}
				}
				for (OWLClass c : n.getEntities()) {
					if (c.equals(rep))
						continue;
					LOG.info(c + " --> "+rep);
					e2iri.put(c, rep.getIRI());
					if (isAddEquivalenceAxioms) {
						// if A is equivalent to B we may wish to retain a historical
						// record of this equivalance - after A is merged into B
						// (assuming B is the representative), all axioms referencing A
						// will be gone, so we re-add the original equivalence,
						// possibly translating to an obo-style xref
						OWLAxiom eca = graph.getDataFactory().getOWLEquivalentClassesAxiom(c, rep);
						if (true) {
							// TODO - allow other options - for now make an xref
							OWLAnnotationProperty lap = graph.getAnnotationProperty(OboFormatTag.TAG_XREF.getTag());
							OWLAnnotationValue value = 
									graph.getDataFactory().getOWLLiteral(graph.getIdentifier(c));
							eca =
									graph.getDataFactory().getOWLAnnotationAssertionAxiom(lap, rep.getIRI(), value);
						}
						else {
							// note: this creates a dangler
							// note: if  |equivalence set| = n>2, creates n-1 axioms 
							
						}
						LOG.info("Preserving ECA to represetative: "+eca);
						newAxioms.add(eca);
					}
				}
			}

			// some properties may be desired to have cardinality = 1
			for (OWLAnnotationProperty p : propertyPrefixScoreMap.keySet()) {
				Map<String, Double> pmap = propertyPrefixScoreMap.get(p);
				OWLClass rep = null;
				Double best = null;
				for (OWLClass c : n.getEntities()) {
					String v = graph.getAnnotationValue(c, p);
					if (v == null || v.equals(""))
						continue;
					LOG.info(c + " . "+p+" = "+v);
					Double score = getScore(c, pmap);
					if (best == null || (score != null && score > best)) {
{'fraction_non_alphanumeric': 0.1249877559016554, 'fraction_numerical': 0.0011754334410813989, 'mean_word_length': 4.002449779519843, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4557248', 'n_tokens_mistral': 4121, 'n_tokens_neox': 3496, 'n_words': 701}