<commit_before>package owltools.sim2;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.Vector;

import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.commons.math3.util.MultidimensionalCounter.Iterator;
import org.apache.log4j.Logger;
import org.semanticweb.elk.owlapi.ElkReasonerFactory;
import org.semanticweb.owlapi.model.IRI;
import org.semanticweb.owlapi.model.OWLClass;
import org.semanticweb.owlapi.model.OWLNamedIndividual;
import org.semanticweb.owlapi.model.OWLNamedObject;
import org.semanticweb.owlapi.model.OWLOntology;
import org.semanticweb.owlapi.reasoner.Node;
import org.semanticweb.owlapi.reasoner.NodeSet;
import org.semanticweb.owlapi.reasoner.OWLReasoner;
import org.semanticweb.owlapi.reasoner.impl.OWLClassNode;

import owltools.sim2.SimpleOwlSim.Direction;
import owltools.sim2.SimpleOwlSim.Metric;
import owltools.sim2.SimpleOwlSim.SimConfigurationProperty;
import owltools.sim2.io.SimResultRenderer.AttributesSimScores;
import owltools.sim2.scores.AttributePairScores;
import owltools.sim2.scores.ElementPairScores;
import owltools.sim2.scores.ScoreMatrix;

import com.googlecode.javaewah.EWAHCompressedBitmap;
import com.googlecode.javaewah.IntIterator;

/**
 * Faster implementation of OwlSim
 * 
 * Makes use of integers to index classes, and bitmaps to represent class sets.
 * 
 * @author cjm
 *
 */
public class FastOwlSim extends AbstractOwlSim implements OwlSim {

	private Logger LOG = Logger.getLogger(FastOwlSim.class);

	public OWLClass debugClass;
	int debugClassMessages = 0;
	int debugClassMessages2 = 0;

	private Map<OWLNamedIndividual, Set<OWLClass>> elementToDirectAttributesMap;

	// -----------
	// CACHES
	// -----------

	// all direct and inferred classes for al individual
	// todo: resolve redundancy with inferredTypesMap
	private Map<OWLNamedIndividual, Set<Node<OWLClass>>> elementToInferredAttributesMap;

	// caches RSub(c) (reflexive subsumers of a class) - NodeSet
	private Map<OWLClass,Set<Node<OWLClass>>> superclassMap;

	// cache of Type(i)->Cs
	private Map<OWLNamedIndividual,Set<Node<OWLClass>>> inferredTypesMap; 

	// cache of RSub(c)->Ints
	private Map<OWLClass,Set<Integer>> superclassIntMap; 
	private Map<OWLNamedIndividual,Set<Integer>> inferredTypesIntMap; // cache of RSub(c)->Ints

	// -------------
	// BITMAP CACHES
	// -------------
	// for efficiency, we store sets of classes as bitmaps.
	// each class is assigned an integer value, resulting
	// it bitmaps of with |c| bits

	// given a class index, return superclasses, as a bitmap
	// e.g. if class C has index 7, then superclassBitmapIndex[7] returns a bitmap,
	// in which every "1" value is the index of a superclass of C
	private EWAHCompressedBitmap[] superclassBitmapIndex;

	// given a class object, return superclasses, as a bitmap. Fast cache for RSub(c)
	private Map<OWLClass,EWAHCompressedBitmap> superclassBitmapMap; 

	// given an individual, return superclasses as a bit map
	private Map<OWLNamedIndividual, EWAHCompressedBitmap> inferredTypesBitmapMap; // cache of Type(i)->BM

	// given a class object, return proper superclasses, as a bitmap. Fast cache for Sub(c)
	Map<OWLClass,EWAHCompressedBitmap> properSuperclassBitmapMap; // cache of Sub(c)->BM

	// -------------
	// CLASS INDICES
	// -------------
	// Each class is assigned a numeric index.
	// We can collapse sets of equivalent classes into a node, which
	// has an arbitrarily assigned representative element.
	//
	// note that the following are for *all* classes in the ontology.
	// E.g. may include anatomy classes in a phenotype analysis.
	// to limit memory usage, first filter ontology to classes with members
	// (being sure to retain inferred axioms).
	// in practice this may not be necessary, as 2D arrays are only used
	// for classes with members.

	// maps a set of equivalent classes to a representative
	Map<Node<OWLClass>, OWLClass> representativeClassMap;

	// maps a class to a representative for that class (typically itself)
	Map<OWLClass, OWLClass> classTorepresentativeClassMap;

	// maps a class to a unique integer
	Map<OWLClass,Integer> classIndex;

	// maps a class index to a class
	OWLClass[] classArray;

	// all types used directly.
	// same as elementToDirectAttributesMap.values()
	private Set<OWLClass> allTypesDirect = null; 

	// all Types used in Type(e) for all e in E.
	// note this excludes classes with no (inferred) members
	// same as elementToInferredAttributesMap.values()
	private Set<OWLClass> allTypesInferred = null; 

	// cache of information content, by class
	private Map<OWLClass, Double> icCache = new HashMap<OWLClass,Double>();

	private Map<OWLClass, Integer> numElementsCache = new HashMap<OWLClass,Integer>();

	// cache of information content, by class index
	Double[] icClassArray = null;

	//	private Map<ClassIntPair, Set<Integer>> classPairLCSMap;
	//	private Map<ClassIntPair, ScoreAttributeSetPair> classPairICLCSMap;

	// used for storing IC values as integers
	final int scaleFactor = 1000;
	//short[][] ciPairScaledScore;
	ScoreAttributeSetPair[][] testCache = null;

	//
	// Symmetric 2D arrays
	//  - for a lookup [cix][dix], ensure cix <= dix
	boolean[][] ciPairIsCached = null;   // determins if there is an entry is ciPairLCS
	int[][] ciPairLCS = null; // LCS ix for a (cix,dix) pair

	// maps an individual to a unique integer
	Map<OWLNamedIndividual,Integer> individualIndex;

	int[][] coaMatrix = null;

	@Override
	public void dispose() {
		showTimings();
	}




	// represents a pair of classes using their indices
	// NOTE; replaced by arrays
	@Deprecated
	private class ClassIntPair {
		int c;
		int d;
		public ClassIntPair(int c, int d) {
			super();
			this.c = c;
			this.d = d;
		}

		@Override
		public int hashCode() {
			final int prime = 991;
			int result = 1;
			result = prime * result + c;
			result = prime * result + d;
			return result;
		}

		@Override
		public boolean equals(Object obj) {
			if (this == obj)
				return true;
			if (obj == null)
				return false;
			if (getClass() != obj.getClass())
				return false;
			ClassIntPair other = (ClassIntPair) obj;
			return c == other.c && d == other.d;
		}
	}



	/**
	 * @param sourceOntology
	 */
	public FastOwlSim(OWLOntology sourceOntology) {
		reasoner = new ElkReasonerFactory().createReasoner(sourceOntology);
	}

	/**
	 * @param reasoner
	 */
	public FastOwlSim(OWLReasoner reasoner) {
		this.reasoner = reasoner;
	}

	@Override
	public Set<OWLClass> getAllAttributeClasses() {
		return allTypesInferred; // note - only attributes used directly or indirectly
	}

	private int getClassIndex(OWLClass c) throws UnknownOWLClassException {
		Integer ix = classIndex.get(c);
		if (ix == null) {
			throw new UnknownOWLClassException(c);
		}
		return ix;
	}
	private int getClassIndex(Node<OWLClass> n) throws UnknownOWLClassException {
		OWLClass c = getDeterministicRepresentative(n);
		return getClassIndex(c);
	}
	private <T extends OWLNamedObject> T getDeterministicRepresentative(Node<T> n) {
		T rep = null;
		// find lexical maximal entry as representative node
		for (T entry : n) {
			if (rep == null) {
				rep = entry;
			}
			else {
				// use IRI for comparison
				IRI repIRI = rep.getIRI();
				IRI entryIRI = entry.getIRI();
				if (entryIRI.compareTo(repIRI) > 0) {
					rep = entry;
				}
			}
		}
		return rep;
	}

	// not yet implemented: guaranteed to yield and indexed class
	private OWLClass getIndexedClass(Node<OWLClass> n) throws UnknownOWLClassException {
		if (representativeClassMap == null)
			representativeClassMap = new HashMap<Node<OWLClass>, OWLClass>();
		else if (representativeClassMap.containsKey(n))
			return representativeClassMap.get(n);
		for (OWLClass c : n.getEntities()) {
			if (classIndex.containsKey(c)) {
				representativeClassMap.put(n,c);
				return c;				
			}
		}
		throw new UnknownOWLClassException(getDeterministicRepresentative(n));
	}

	/* (non-Javadoc)
	 * @see owltools.sim2.OwlSim#createElementAttributeMapFromOntology()
	 */
	@Override
	public void createElementAttributeMapFromOntology() throws UnknownOWLClassException {
		getReasoner().flush();

		// cache E -> Type(E)
		elementToDirectAttributesMap = new HashMap<OWLNamedIndividual,Set<OWLClass>>();
		elementToInferredAttributesMap = new HashMap<OWLNamedIndividual,Set<Node<OWLClass>>>();
		allTypesDirect = new HashSet<OWLClass>();
		allTypesInferred = new HashSet<OWLClass>();
		Set<OWLNamedIndividual> inds = getSourceOntology().getIndividualsInSignature(true);
		for (OWLNamedIndividual e : inds) {

			// The attribute classes for an individual are the direct inferred
			// named types. We assume that grouping classes have already been
			// generated.
			NodeSet<OWLClass> nodesetDirect = getReasoner().getTypes(e, true);
			NodeSet<OWLClass> nodesetInferred = getReasoner().getTypes(e, false);
			allTypesDirect.addAll(nodesetDirect.getFlattened());
			allTypesInferred.addAll(nodesetInferred.getFlattened());
			elementToDirectAttributesMap.put(e, nodesetDirect.getFlattened());
			elementToInferredAttributesMap.put(e, nodesetInferred.getNodes());

			// TODO - remove deprecated classes
			elementToDirectAttributesMap.get(e).remove(owlThing());
			elementToInferredAttributesMap.get(e).remove(owlThing());
		}

		LOG.info("|TypesUsedDirectly|="+allTypesDirect.size());
		LOG.info("|TypesUsedInferred|="+allTypesInferred.size());

		if (this.getPropertyAsBoolean(SimConfigurationProperty.useAllClasses)) {
			allTypesInferred = getSourceOntology().getClassesInSignature(true);
		}

		Set<OWLClass> cset = allTypesInferred;

		LOG.info("|C|="+cset.size());
		LOG.info("|I|="+inds.size());

		Set<Node<OWLClass>> nodes = new HashSet<Node<OWLClass>>();
		for (OWLClass c : cset) {
			nodes.add(getReasoner().getEquivalentClasses(c));
		}
		LOG.info("|N|="+nodes.size()); // TODO - use this

		// classes are collapsed into nodes. Create a map from node to
		// class, and a map of every class
		// to the representative element from its node
		representativeClassMap = new HashMap<Node<OWLClass>, OWLClass>();
		classTorepresentativeClassMap = new HashMap<OWLClass, OWLClass>();
		for (Node<OWLClass> n : nodes) {
			OWLClass c = getDeterministicRepresentative(n);
			representativeClassMap.put(n, c);
			for (OWLClass c2 : n.getEntities()) {
				classTorepresentativeClassMap.put(c2, c);
			}
		}

		// Create a bidirectional index, class by number
		int n=0;
		classArray = (OWLClass[]) Array.newInstance(OWLClass.class, cset.size()+1);
		classIndex = new HashMap<OWLClass,Integer>();

		// 0th element is always owlThing (root)
		classArray[n] = owlThing();
		classIndex.put(owlThing(), n);
		n++;

		// TODO - investigate if ordering elements makes a difference;
		// e.g. if more frequent classes receive lower bit indices this
		// may speed certain BitMap operations?
		for (OWLClass c : cset) {
			if (c.equals(owlThing()))
				continue;
			classArray[n] = c;
			classIndex.put(c, n);
			n++;
		}

		// ensure cached
		cset.add(owlThing());
		for (OWLClass c : cset) {
			ancsCachedModifiable(c);
			ancsIntsCachedModifiable(c);
			ancsBitmapCachedModifiable(c);
		}
		for (OWLNamedIndividual e : inds) {
			// force cacheing
			ancsBitmapCachedModifiable(e);
		}



		// cache - this is for all types inferred
		for (OWLClass c : cset) {
			getInformationContentForAttribute(c);
			getInformationContentForAttribute(classIndex.get(c));
		}
		this.assignDefaultInformationContentForAllClasses();

		this.computeSystemStats();

	}

	// TODO - change set to be (ordered) List, to avoid sorting each time
	private EWAHCompressedBitmap convertIntsToBitmap(Set<Integer> bits) {
		EWAHCompressedBitmap bm = new EWAHCompressedBitmap();
		ArrayList<Integer> bitlist = new ArrayList<Integer>(bits);
		// necessary for EWAH API, otherwise silently fails
		Collections.sort(bitlist);
		for (Integer i : bitlist) {
			bm.set(i.intValue());
		}
		return bm;
	}

	// cached proper superclasses (i.e. excludes equivalent classes) as BitMap
	private EWAHCompressedBitmap ancsProperBitmapCachedModifiable(OWLClass c) {
		if (properSuperclassBitmapMap != null && properSuperclassBitmapMap.containsKey(c)) {
			return properSuperclassBitmapMap.get(c);
		}

		Set<Integer> ancsInts = new HashSet<Integer>();
		for (Node<OWLClass> anc : reasoner.getSuperClasses(c, false)) {
			// TODO - verify robust for non-Rep elements<commit_msg>Adding additional logging to determine phenoscape issue<commit_after>package owltools.sim2;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.Vector;

import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.commons.math3.util.MultidimensionalCounter.Iterator;
import org.apache.log4j.Logger;
import org.semanticweb.elk.owlapi.ElkReasonerFactory;
import org.semanticweb.owlapi.model.IRI;
import org.semanticweb.owlapi.model.OWLClass;
import org.semanticweb.owlapi.model.OWLNamedIndividual;
import org.semanticweb.owlapi.model.OWLNamedObject;
import org.semanticweb.owlapi.model.OWLOntology;
import org.semanticweb.owlapi.reasoner.Node;
import org.semanticweb.owlapi.reasoner.NodeSet;
import org.semanticweb.owlapi.reasoner.OWLReasoner;
import org.semanticweb.owlapi.reasoner.impl.OWLClassNode;

import owltools.sim2.SimpleOwlSim.Direction;
import owltools.sim2.SimpleOwlSim.Metric;
import owltools.sim2.SimpleOwlSim.SimConfigurationProperty;
import owltools.sim2.io.SimResultRenderer.AttributesSimScores;
import owltools.sim2.scores.AttributePairScores;
import owltools.sim2.scores.ElementPairScores;
import owltools.sim2.scores.ScoreMatrix;

import com.googlecode.javaewah.EWAHCompressedBitmap;
import com.googlecode.javaewah.IntIterator;

/**
 * Faster implementation of OwlSim
 * 
 * Makes use of integers to index classes, and bitmaps to represent class sets.
 * 
 * @author cjm
 *
 */
public class FastOwlSim extends AbstractOwlSim implements OwlSim {

	private Logger LOG = Logger.getLogger(FastOwlSim.class);

	public OWLClass debugClass;
	int debugClassMessages = 0;
	int debugClassMessages2 = 0;

	private Map<OWLNamedIndividual, Set<OWLClass>> elementToDirectAttributesMap;

	// -----------
	// CACHES
	// -----------

	// all direct and inferred classes for al individual
	// todo: resolve redundancy with inferredTypesMap
	private Map<OWLNamedIndividual, Set<Node<OWLClass>>> elementToInferredAttributesMap;

	// caches RSub(c) (reflexive subsumers of a class) - NodeSet
	private Map<OWLClass,Set<Node<OWLClass>>> superclassMap;

	// cache of Type(i)->Cs
	private Map<OWLNamedIndividual,Set<Node<OWLClass>>> inferredTypesMap; 

	// cache of RSub(c)->Ints
	private Map<OWLClass,Set<Integer>> superclassIntMap; 
	private Map<OWLNamedIndividual,Set<Integer>> inferredTypesIntMap; // cache of RSub(c)->Ints

	// -------------
	// BITMAP CACHES
	// -------------
	// for efficiency, we store sets of classes as bitmaps.
	// each class is assigned an integer value, resulting
	// it bitmaps of with |c| bits

	// given a class index, return superclasses, as a bitmap
	// e.g. if class C has index 7, then superclassBitmapIndex[7] returns a bitmap,
	// in which every "1" value is the index of a superclass of C
	private EWAHCompressedBitmap[] superclassBitmapIndex;

	// given a class object, return superclasses, as a bitmap. Fast cache for RSub(c)
	private Map<OWLClass,EWAHCompressedBitmap> superclassBitmapMap; 

	// given an individual, return superclasses as a bit map
	private Map<OWLNamedIndividual, EWAHCompressedBitmap> inferredTypesBitmapMap; // cache of Type(i)->BM

	// given a class object, return proper superclasses, as a bitmap. Fast cache for Sub(c)
	Map<OWLClass,EWAHCompressedBitmap> properSuperclassBitmapMap; // cache of Sub(c)->BM

	// -------------
	// CLASS INDICES
	// -------------
	// Each class is assigned a numeric index.
	// We can collapse sets of equivalent classes into a node, which
	// has an arbitrarily assigned representative element.
	//
	// note that the following are for *all* classes in the ontology.
	// E.g. may include anatomy classes in a phenotype analysis.
	// to limit memory usage, first filter ontology to classes with members
	// (being sure to retain inferred axioms).
	// in practice this may not be necessary, as 2D arrays are only used
	// for classes with members.

	// maps a set of equivalent classes to a representative
	Map<Node<OWLClass>, OWLClass> representativeClassMap;

	// maps a class to a representative for that class (typically itself)
	Map<OWLClass, OWLClass> classTorepresentativeClassMap;

	// maps a class to a unique integer
	Map<OWLClass,Integer> classIndex;

	// maps a class index to a class
	OWLClass[] classArray;

	// all types used directly.
	// same as elementToDirectAttributesMap.values()
	private Set<OWLClass> allTypesDirect = null; 

	// all Types used in Type(e) for all e in E.
	// note this excludes classes with no (inferred) members
	// same as elementToInferredAttributesMap.values()
	private Set<OWLClass> allTypesInferred = null; 

	// cache of information content, by class
	private Map<OWLClass, Double> icCache = new HashMap<OWLClass,Double>();

	private Map<OWLClass, Integer> numElementsCache = new HashMap<OWLClass,Integer>();

	// cache of information content, by class index
	Double[] icClassArray = null;

	//	private Map<ClassIntPair, Set<Integer>> classPairLCSMap;
	//	private Map<ClassIntPair, ScoreAttributeSetPair> classPairICLCSMap;

	// used for storing IC values as integers
	final int scaleFactor = 1000;
	//short[][] ciPairScaledScore;
	ScoreAttributeSetPair[][] testCache = null;

	//
	// Symmetric 2D arrays
	//  - for a lookup [cix][dix], ensure cix <= dix
	boolean[][] ciPairIsCached = null;   // determins if there is an entry is ciPairLCS
	int[][] ciPairLCS = null; // LCS ix for a (cix,dix) pair

	// maps an individual to a unique integer
	Map<OWLNamedIndividual,Integer> individualIndex;

	int[][] coaMatrix = null;

	@Override
	public void dispose() {
		showTimings();
	}




	// represents a pair of classes using their indices
	// NOTE; replaced by arrays
	@Deprecated
	private class ClassIntPair {
		int c;
		int d;
		public ClassIntPair(int c, int d) {
			super();
			this.c = c;
			this.d = d;
		}

		@Override
		public int hashCode() {
			final int prime = 991;
			int result = 1;
			result = prime * result + c;
			result = prime * result + d;
			return result;
		}

		@Override
		public boolean equals(Object obj) {
			if (this == obj)
				return true;
			if (obj == null)
				return false;
			if (getClass() != obj.getClass())
				return false;
			ClassIntPair other = (ClassIntPair) obj;
			return c == other.c && d == other.d;
		}
	}



	/**
	 * @param sourceOntology
	 */
	public FastOwlSim(OWLOntology sourceOntology) {
		reasoner = new ElkReasonerFactory().createReasoner(sourceOntology);
	}

	/**
	 * @param reasoner
	 */
	public FastOwlSim(OWLReasoner reasoner) {
		this.reasoner = reasoner;
	}

	@Override
	public Set<OWLClass> getAllAttributeClasses() {
		return allTypesInferred; // note - only attributes used directly or indirectly
	}

	private int getClassIndex(OWLClass c) throws UnknownOWLClassException {
		Integer ix = classIndex.get(c);
		if (ix == null) {
			throw new UnknownOWLClassException(c);
		}
		return ix;
	}
	private int getClassIndex(Node<OWLClass> n) throws UnknownOWLClassException {
		OWLClass c = getDeterministicRepresentative(n);
		return getClassIndex(c);
	}
	private <T extends OWLNamedObject> T getDeterministicRepresentative(Node<T> n) {
		T rep = null;
		// find lexical maximal entry as representative node
		for (T entry : n) {
			if (rep == null) {
				rep = entry;
			}
			else {
				// use IRI for comparison
				IRI repIRI = rep.getIRI();
				IRI entryIRI = entry.getIRI();
				if (entryIRI.compareTo(repIRI) > 0) {
					rep = entry;
				}
			}
		}
		return rep;
	}

	// not yet implemented: guaranteed to yield and indexed class
	private OWLClass getIndexedClass(Node<OWLClass> n) throws UnknownOWLClassException {
		if (representativeClassMap == null)
			representativeClassMap = new HashMap<Node<OWLClass>, OWLClass>();
		else if (representativeClassMap.containsKey(n))
			return representativeClassMap.get(n);
		for (OWLClass c : n.getEntities()) {
			if (classIndex.containsKey(c)) {
				representativeClassMap.put(n,c);
				return c;				
			}
		}
		throw new UnknownOWLClassException(getDeterministicRepresentative(n));
	}

	/* (non-Javadoc)
	 * @see owltools.sim2.OwlSim#createElementAttributeMapFromOntology()
	 */
	@Override
	public void createElementAttributeMapFromOntology() throws UnknownOWLClassException {
		getReasoner().flush();

		// cache E -> Type(E)
		elementToDirectAttributesMap = new HashMap<OWLNamedIndividual,Set<OWLClass>>();
		elementToInferredAttributesMap = new HashMap<OWLNamedIndividual,Set<Node<OWLClass>>>();
		allTypesDirect = new HashSet<OWLClass>();
		allTypesInferred = new HashSet<OWLClass>();
		Set<OWLNamedIndividual> inds = getSourceOntology().getIndividualsInSignature(true);
		for (OWLNamedIndividual e : inds) {

			// The attribute classes for an individual are the direct inferred
			// named types. We assume that grouping classes have already been
			// generated.
			NodeSet<OWLClass> nodesetDirect = getReasoner().getTypes(e, true);
			NodeSet<OWLClass> nodesetInferred = getReasoner().getTypes(e, false);
			allTypesDirect.addAll(nodesetDirect.getFlattened());
			allTypesInferred.addAll(nodesetInferred.getFlattened());
			elementToDirectAttributesMap.put(e, nodesetDirect.getFlattened());
			elementToInferredAttributesMap.put(e, nodesetInferred.getNodes());

			// TODO - remove deprecated classes
			elementToDirectAttributesMap.get(e).remove(owlThing());
			elementToInferredAttributesMap.get(e).remove(owlThing());
		}

		LOG.info("|TypesUsedDirectly|="+allTypesDirect.size());
		LOG.info("|TypesUsedInferred|="+allTypesInferred.size());

		if (this.getPropertyAsBoolean(SimConfigurationProperty.useAllClasses)) {
			allTypesInferred = getSourceOntology().getClassesInSignature(true);
		}

		Set<OWLClass> cset = allTypesInferred;

		LOG.info("|C|="+cset.size());
		LOG.info("|I|="+inds.size());

		Set<Node<OWLClass>> nodes = new HashSet<Node<OWLClass>>();
		for (OWLClass c : cset) {
			nodes.add(getReasoner().getEquivalentClasses(c));
		}
		LOG.info("|N|="+nodes.size()); // TODO - use this

		// classes are collapsed into nodes. Create a map from node to
		// class, and a map of every class
		// to the representative element from its node
		representativeClassMap = new HashMap<Node<OWLClass>, OWLClass>();
		classTorepresentativeClassMap = new HashMap<OWLClass, OWLClass>();
		for (Node<OWLClass> n : nodes) {
			OWLClass c = getDeterministicRepresentative(n);
			representativeClassMap.put(n, c);
			for (OWLClass c2 : n.getEntities()) {
				classTorepresentativeClassMap.put(c2, c);
			}
		}

		// Create a bidirectional index, class by number
		int n=0;
		classArray = (OWLClass[]) Array.newInstance(OWLClass.class, cset.size()+1);
		classIndex = new HashMap<OWLClass,Integer>();

		// 0th element is always owlThing (root)
		classArray[n] = owlThing();
		classIndex.put(owlThing(), n);
		n++;

		// TODO - investigate if ordering elements makes a difference;
		// e.g. if more frequent classes receive lower bit indices this
		// may speed certain BitMap operations?
		for (OWLClass c : cset) {
			if (c.equals(owlThing()))
				continue;
			classArray[n] = c;
			classIndex.put(c, n);
			n++;
		}

		// ensure cached
		cset.add(owlThing());
		for (OWLClass c : cset) {
			ancsCachedModifiable(c);
			ancsIntsCachedModifiable(c);
			ancsBitmapCachedModifiable(c);
		}
		for (OWLNamedIndividual e : inds) {
			// force cacheing
			ancsBitmapCachedModifiable(e);
		}



		// cache - this is for all types inferred
		for (OWLClass c : cset) {
			getInformationContentForAttribute(c);
			getInformationContentForAttribute(classIndex.get(c));
		}
		LOG.info("Done computing ICs for all classes");
		this.assignDefaultInformationContentForAllClasses();
		LOG.info("Done assigning default IC");

		this.computeSystemStats();

	}

	// TODO - change set to be (ordered) List, to avoid sorting each time
	private EWAHCompressedBitmap convertIntsToBitmap(Set<Integer> bits) {
		EWAHCompressedBitmap bm = new EWAHCompressedBitmap();
		ArrayList<Integer> bitlist = new ArrayList<Integer>(bits);
		// necessary for EWAH API, otherwise silently fails
		Collections.sort(bitlist);
		for (Integer i : bitlist) {
			bm.set(i.intValue());
		}
		return bm;
	}

	// cached proper superclasses (i.e. excludes equivalent classes) as BitMap
	private EWAHCompressedBitmap ancsProperBitmapCachedModifiable(OWLClass c) {
		if (properSuperclassBitmapMap != null && properSuperclassBitmapMap.containsKey(c)) {
			return properSuperclassBitmapMap.get(c);
		}

		Set<Integer> ancsInts = new HashSet<Integer>();
		for (Node<OWLClass> anc : reasoner.getSuperClasses(c, false)) {
			// TODO - verify robust for non-Rep elements
{'fraction_non_alphanumeric': 0.11322078833482786, 'fraction_numerical': 0.002651175484424344, 'mean_word_length': 4.235762400489896, 'pattern_counts': {'":': 0, '<': 135, '<?xml version=': 0, '>': 147, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4906490', 'n_tokens_mistral': 8952, 'n_tokens_neox': 8113, 'n_words': 2157}