<commit_before>package owltools.gaf.lego;

import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.log4j.Logger;
import org.coode.owlapi.manchesterowlsyntax.ManchesterOWLSyntaxOntologyFormat;
import org.geneontology.lego.dot.LegoDotWriter;
import org.geneontology.lego.dot.LegoRenderer;
import org.geneontology.lego.model.LegoTools.UnExpectedStructureException;
import org.semanticweb.owlapi.expression.ParserException;
import org.semanticweb.owlapi.io.OWLFunctionalSyntaxOntologyFormat;
import org.semanticweb.owlapi.io.OWLOntologyDocumentSource;
import org.semanticweb.owlapi.io.OWLXMLOntologyFormat;
import org.semanticweb.owlapi.io.RDFXMLOntologyFormat;
import org.semanticweb.owlapi.io.StringDocumentSource;
import org.semanticweb.owlapi.model.AddAxiom;
import org.semanticweb.owlapi.model.AddImport;
import org.semanticweb.owlapi.model.AddOntologyAnnotation;
import org.semanticweb.owlapi.model.AxiomType;
import org.semanticweb.owlapi.model.IRI;
import org.semanticweb.owlapi.model.OWLAnnotation;
import org.semanticweb.owlapi.model.OWLAnnotationAssertionAxiom;
import org.semanticweb.owlapi.model.OWLAnnotationProperty;
import org.semanticweb.owlapi.model.OWLAnnotationValue;
import org.semanticweb.owlapi.model.OWLAxiom;
import org.semanticweb.owlapi.model.OWLAxiomChange;
import org.semanticweb.owlapi.model.OWLClass;
import org.semanticweb.owlapi.model.OWLClassAssertionAxiom;
import org.semanticweb.owlapi.model.OWLClassExpression;
import org.semanticweb.owlapi.model.OWLDataFactory;
import org.semanticweb.owlapi.model.OWLEntity;
import org.semanticweb.owlapi.model.OWLException;
import org.semanticweb.owlapi.model.OWLImportsDeclaration;
import org.semanticweb.owlapi.model.OWLIndividual;
import org.semanticweb.owlapi.model.OWLNamedIndividual;
import org.semanticweb.owlapi.model.OWLObjectProperty;
import org.semanticweb.owlapi.model.OWLObjectPropertyAssertionAxiom;
import org.semanticweb.owlapi.model.OWLObjectPropertyExpression;
import org.semanticweb.owlapi.model.OWLObjectSomeValuesFrom;
import org.semanticweb.owlapi.model.OWLOntology;
import org.semanticweb.owlapi.model.OWLOntologyAlreadyExistsException;
import org.semanticweb.owlapi.model.OWLOntologyChange;
import org.semanticweb.owlapi.model.OWLOntologyChangeVisitor;
import org.semanticweb.owlapi.model.OWLOntologyCreationException;
import org.semanticweb.owlapi.model.OWLOntologyDocumentAlreadyExistsException;
import org.semanticweb.owlapi.model.OWLOntologyFormat;
import org.semanticweb.owlapi.model.OWLOntologyID;
import org.semanticweb.owlapi.model.OWLOntologyIRIMapper;
import org.semanticweb.owlapi.model.OWLOntologyManager;
import org.semanticweb.owlapi.model.OWLOntologyStorageException;
import org.semanticweb.owlapi.model.RemoveAxiom;
import org.semanticweb.owlapi.model.RemoveImport;
import org.semanticweb.owlapi.model.RemoveOntologyAnnotation;
import org.semanticweb.owlapi.model.SetOntologyID;
import org.semanticweb.owlapi.reasoner.OWLReasoner;
import org.semanticweb.owlapi.vocab.OWLRDFVocabulary;

import owltools.gaf.GafDocument;
import owltools.gaf.bioentities.ProteinTools;
import owltools.gaf.eco.EcoMapper;
import owltools.gaf.parser.GafObjectsBuilder;
import owltools.graph.OWLGraphWrapper;
import owltools.io.CatalogXmlIRIMapper;
import owltools.vocab.OBOUpperVocabulary;

/**
 * Manager object for operations on collections of MolecularModels (aka lego diagrams)
 * 
 * any number of models can be loaded at any time (todo - impose some limit to avoid
 * using too much memory)
 * 
 * each model has a generator, an OWLOntology (containing the set of class assertions)
 * and a reasoner associated with it (todo - test memory requirements)
 * 
 * This manager is designed to be used within a web server. Multiple clients can
 * contact the same manager instance through services
 *
 */
public class MolecularModelManager {
	
	private static Logger LOG = Logger.getLogger(MolecularModelManager.class);


	OWLGraphWrapper graph;
	boolean isPrecomputePropertyClassCombinations = false;
	Map<String, GafDocument> dbToGafdoc = new HashMap<String, GafDocument>();
	Map<String, LegoModelGenerator> modelMap = new HashMap<String, LegoModelGenerator>();
	String pathToGafs = "gene-associations";
	String pathToOWLFiles = "owl-models";
	String pathToProteinFiles = null;
	Map<String, String> dbToTaxon = null;
	OWLOntologyIRIMapper proteinMapper = null;
	
	GafObjectsBuilder builder = new GafObjectsBuilder();
	// WARNING: Do *NOT* switch to functional syntax until the OWL-API has fixed a bug.
	OWLOntologyFormat ontologyFormat = new ManchesterOWLSyntaxOntologyFormat();
	Set<IRI> additionalImports;
	final Set<IRI> obsoleteImports = new HashSet<IRI>();

	// TODO: Temporarily for keeping instances unique (search for "unique" below).
	static String uniqueTop = Long.toHexString((System.currentTimeMillis()/1000));
	static long instanceCounter = 0;
	private static String localUnique(){
		instanceCounter++;
		String unique = uniqueTop + String.format("%07d", instanceCounter);
		return unique;		
	}

	/**
	 * Represents the reponse to a requested translation on an
	 * ontology/model
	 * 
	 */
	public class OWLOperationResponse {
		OWLAxiomChange change;
		int changeId;
		boolean isSuccess = true;
		boolean isResultsInInconsistency = false;
		Object modelData = null;
		List<OWLNamedIndividual> individuals = null;
		
		/**
		 * @param isSuccess
		 */
		public OWLOperationResponse(boolean isSuccess) {
			super();
			this.isSuccess = isSuccess;
		}
		
		/**
		 * @param isSuccess
		 * @param isResultsInInconsistency
		 */
		public OWLOperationResponse(boolean isSuccess,
				boolean isResultsInInconsistency) {
			super();
			this.isSuccess = isSuccess;
			this.isResultsInInconsistency = isResultsInInconsistency;
		}
		
		public OWLOperationResponse(OWLAxiomChange change, boolean isSuccess,
				boolean isResultsInInconsistency) {
			super();
			this.isSuccess = isSuccess;
			this.isResultsInInconsistency = isResultsInInconsistency;
			this.change = change;
		}
		
		/**
		 * @return the isSuccess
		 */
		public boolean isSuccess() {
			return isSuccess;
		}
		
		/**
		 * @return the isResultsInInconsistency
		 */
		public boolean isResultsInInconsistency() {
			return isResultsInInconsistency;
		}
		
		/**
		 * @return the modelData
		 */
		public Object getModelData() {
			return modelData;
		}
		
		/**
		 * @param modelData the modelData to set
		 */
		public void setModelData(Object modelData) {
			this.modelData = modelData;
		}
		
		/**
		 * @return the individuals
		 */
		public List<OWLNamedIndividual> getIndividuals() {
			return individuals;
		}

		/**
		 * @param individuals the individuals to set
		 */
		public void setIndividuals(List<OWLNamedIndividual> individuals) {
			this.individuals = individuals;
		}
		
		
//		/**
//		 * Combine two responses together.
//		 * 
//		 * @param response
//		 */
//		public void mergeIn(OWLOperationResponse toMerge) {
//
//			// success and inconsistency dominate false and true respectively.
//			if( ! toMerge.isSuccess ) this.isSuccess = false;
//			if( ! toMerge.isResultsInInconsistency() ) this.isResultsInInconsistency = true;
//			
//			// Merge model data.
//			Map<Object, Object> newMap = new HashMap<Object,Object>();
//			List<Map<Object, Object>> toLoop = this.getModelData(); // add this response
//			toLoop.addAll(toMerge.getModelData()); // add merging response
//			for( Map<Object, Object>mergable : toLoop){
//				// If new, add to new.
//				if( mergable.containsKey("id") ){
//					
//				}
//			}
//			
//			// Merge ID list via Set (no dupes).
//		    Set<String> set = new HashSet<String>();
//		    set.addAll(this.getIndividualIds());
//		    set.addAll(toMerge.getIndividualIds());
//			this.individualIds = new ArrayList<String>(set);
//		}
		
	}

	public static class UnknownIdentifierException extends Exception {

		// generated
		private static final long serialVersionUID = -847970910712518838L;

		/**
		 * @param message
		 * @param cause
		 */
		public UnknownIdentifierException(String message, Throwable cause) {
			super(message, cause);
		}

		/**
		 * @param message
		 */
		public UnknownIdentifierException(String message) {
			super(message);
		}

	}
	
	/**
	 * @param graph
	 * @throws OWLOntologyCreationException
	 */
	public MolecularModelManager(OWLGraphWrapper graph) throws OWLOntologyCreationException {
		super();
		this.graph = graph;
		init();
	}
	/**
	 * @param ont
	 * @throws OWLOntologyCreationException
	 */
	public MolecularModelManager(OWLOntology ont) throws OWLOntologyCreationException {
		super();
		this.graph = new OWLGraphWrapper(ont);
		init();
	}

	/**
	 * @throws OWLOntologyCreationException
	 */
	protected void init() throws OWLOntologyCreationException {
		// set default imports
		additionalImports = new HashSet<IRI>();
		additionalImports.add(IRI.create("http://purl.obolibrary.org/obo/ro.owl"));
		additionalImports.add(IRI.create("http://purl.obolibrary.org/obo/go/extensions/ro_pending.owl"));
		additionalImports.add(EcoMapper.ECO_PURL_IRI);
	}


	/**
	 * @return graph wrapper for core/source ontology
	 */
	public OWLGraphWrapper getGraph() {
		return graph;
	}

	/**
	 * @return core/source ontology
	 */
	public OWLOntology getOntology() {
		return graph.getSourceOntology();
	}


	/**
	 * @return path to gafs direcory
	 */
	public String getPathToGafs() {
		return pathToGafs;
	}
	/**
	 * Can either be an HTTP prefix, or an absolute file path
	 * 
	 * @param pathToGafs
	 */
	public void setPathToGafs(String pathToGafs) {
		this.pathToGafs = pathToGafs;
	}
	
	/**
	 * Note this may move to an implementation-specific subclass in future
	 * 
	 * @return path to owl on server
	 */
	public String getPathToOWLFiles() {
		return pathToOWLFiles;
	}
	/**
	 * @param pathToOWLFiles
	 */
	public void setPathToOWLFiles(String pathToOWLFiles) {
		this.pathToOWLFiles = pathToOWLFiles;
	}
	
	/**
	 * @param pathToProteinFiles
	 * @throws IOException 
	 */
	public synchronized void setPathToProteinFiles(String pathToProteinFiles) throws IOException {
		setPathToProteinFiles(pathToProteinFiles, "catalog-v001.xml");
	}
	
	/**
	 * @param pathToProteinFiles
	 * @param catalogXML
	 * @throws IOException 
	 */
	public synchronized void setPathToProteinFiles(String pathToProteinFiles, String catalogXML) throws IOException {
		if (proteinMapper != null) {
			graph.getManager().removeIRIMapper(proteinMapper);
		}
		this.pathToProteinFiles = pathToProteinFiles;
		proteinMapper = new CatalogXmlIRIMapper(new File(pathToProteinFiles, catalogXML));
		graph.getManager().addIRIMapper(proteinMapper);
	}
	
	public String getPathToProteinFiles() {
		return pathToProteinFiles;
	}
	
	/**
	 * @return the dbToTaxon
	 */
	public Map<String, String> getDbToTaxon() {
		return dbToTaxon;
	}
	/**
	 * @param dbToTaxon the dbToTaxon to set
	 */
	public void setDbToTaxon(Map<String, String> dbToTaxon) {
		this.dbToTaxon = dbToTaxon;
	}
	/**
	 * loads/register a Gaf document
	 * 
	 * @param db
	 * @return Gaf document
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public GafDocument loadGaf(String db) throws IOException, URISyntaxException {
		if (!dbToGafdoc.containsKey(db)) {
			LOG.info("Loading GAF for db: "+db);
			GafDocument gafdoc = builder.buildDocument(pathToGafs + "/gene_association." + db + ".gz");
			dbToGafdoc.put(db, gafdoc);
		}
		return dbToGafdoc.get(db);
	}

	/**
	 * Loads and caches a GAF document from a specified location
	 * 
	 * @param db
	 * @param gafFile
	 * @return Gaf document
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public GafDocument loadGaf(String db, File gafFile) throws IOException, URISyntaxException {
		if (!dbToGafdoc.containsKey(db)) {

			GafDocument gafdoc = builder.buildDocument(gafFile);
			dbToGafdoc.put(db, gafdoc);
		}
		return dbToGafdoc.get(db);
	}


	/**
	 * @param db
	 * @return Gaf document for db
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public GafDocument getGaf(String db) throws IOException, URISyntaxException {
		return loadGaf(db);
	}

	/**
	 * Add additional import declarations for any newly generated model.
	 * 
	 * @param imports
	 */
	public void addImports(Iterable<String> imports) {
		if (imports != null) {
			for (String importIRIString : imports) {
				additionalImports.add(IRI.create(importIRIString));
			}
		}
	}
	
	/**
	 * Mark the given imports as obsolete.
	 * 
	 * @param obsoletes
	 */
	public void addObsoleteImports(Iterable<String> obsoletes) {
		if (obsoletes != null) {
			for (String obsolete : obsoletes) {
				obsoleteImports.add(IRI.create(obsolete));
			}
		}
	}
	
	public Collection<IRI> getImports() {
		return Collections.unmodifiableCollection(additionalImports);
	}

	/**
	 * Generates a new model taking as input a biological process P and a database D.
	 * The axioms from P and annotations to P from D are used to seed a new model
	 * 
	 * See {@link LegoModelGenerator#buildNetwork(OWLClass, java.util.Collection)}
	 * And also https://docs.google.com/document/d/1TV8Eb9sSvFY-weVZaIfzCxF1qbnmkUaiUhTm9Bs3gRE/edit
	 * 
	 * Note the resulting model is uniquely identified by the modeId, which is currently constructed
	 * as a concatenation of the db and the P id. This means that if there is an existing model by
	 * this ID it will be overwritten
	 * 
	 * @param processCls
	 * @param db
	 * @return modelId
	 * @throws OWLOntologyCreationException
	 * @throws URISyntaxException 
	 * @throws IOException 
	 */
	public String generateModel(OWLClass processCls, String db) throws OWLOntologyCreationException, IOException, URISyntaxException {
		// quick check, only generate a model if it does not already exists.
		final String p = graph.getIdentifier(processCls);
		String modelId = getModelId(p, db);
		if (modelMap.containsKey(modelId)) {
			throw new OWLOntologyCreationException("A model already exists for this process and db: "+modelId);
		}
		LOG.info("Generating model for Class: "+p+" and db: "+db);
		OWLOntology tbox = graph.getSourceOntology();
		OWLOntology abox = null;
		LegoModelGenerator model = null;
		
		// create empty ontology
		// use model id as ontology IRI
		OWLOntologyManager m = graph.getManager();
		IRI iri = MolecularModelJsonRenderer.getIRI(modelId, graph);
		try {
			abox = m.createOntology(iri);
			
			// setup model ontology to import the source ontology and other imports
			createImports(abox, tbox.getOntologyID(), db);
			
			// create generator
			model = new LegoModelGenerator(tbox, abox);
			
			model.setPrecomputePropertyClassCombinations(isPrecomputePropertyClassCombinations);
			Set<String> seedGenes = new HashSet<String>();
			// only look for genes if a GAF is available
			if (db != null) {
				GafDocument gafdoc = getGaf(db);
				model.initialize(gafdoc, graph);
				seedGenes.addAll(model.getGenes(processCls));
			}
			model.setContextualizingSuffix(db);
			model.buildNetwork(p, seedGenes);
		}
		catch (OWLOntologyCreationException exception) {
			if (model != null) {
				model.dispose();
			} else if (abox != null) {
				m.removeOntology(abox);
			}
			throw exception;
		}
		catch (IOException exception) {
			if (model != null) {
				model.dispose();
			}
			throw exception;
		}
		catch (URISyntaxException exception) {
			if (model != null) {
				model.dispose();
			}
			throw exception;
		}
		modelMap.put(modelId, model);
		return modelId;

	}

	private void createImports(OWLOntology ont, OWLOntologyID tboxId, String db) throws OWLOntologyCreationException {
		OWLOntologyManager m = ont.getOWLOntologyManager();
		OWLDataFactory f = m.getOWLDataFactory();
		
		// import T-Box
		OWLImportsDeclaration tBoxImportDeclaration = f.getOWLImportsDeclaration(tboxId.getOntologyIRI());
		m.applyChange(new AddImport(ont, tBoxImportDeclaration));
		
		// import additional ontologies via IRI
		for (IRI importIRI : additionalImports) {
			OWLImportsDeclaration importDeclaration = f.getOWLImportsDeclaration(importIRI);
			// check that the import ontology is available
			OWLOntology importOntology = m.getOntology(importIRI);
			if (importOntology == null) {
				// only try to load it, if it isn't already loaded
				try {
					m.loadOntology(importIRI);
				} catch (OWLOntologyDocumentAlreadyExistsException e) {
					// ignore
				} catch (OWLOntologyAlreadyExistsException e) {
					// ignore
				}
			}
			m.applyChange(new AddImport(ont, importDeclaration));
		}
		
		// check for protein ontology
		if (db != null && pathToProteinFiles != null && proteinMapper != null) {
			String taxonId = null;
			if (dbToTaxon != null) {
				taxonId = dbToTaxon.get(db);
			}
			if (taxonId == null) {
				// fallback
				taxonId = db;
			}
			IRI proteinIRI = ProteinTools.createProteinOntologyIRI(taxonId);
			IRI mapped = proteinMapper.getDocumentIRI(proteinIRI);
			if (mapped != null) {
				OWLImportsDeclaration importDeclaration = f.getOWLImportsDeclaration(proteinIRI);
				m.loadOntology(proteinIRI);
				m.applyChange(new AddImport(ont, importDeclaration));
			}
		}
	}
	
	/**
	 * wrapper for {@link #generateModel(OWLClass, String)}
	 * 
	 * @param pid
	 * @param db
	 * @return modelId
	 * @throws OWLOntologyCreationException
	 * @throws IOException
	 * @throws URISyntaxException
	 * @throws UnknownIdentifierException
	 */
	public String generateModel(String pid, String db) throws OWLOntologyCreationException, IOException, URISyntaxException, UnknownIdentifierException {
		OWLClass cls = getClass(pid, graph);
		if (cls == null) {
			throw new UnknownIdentifierException("Could not find a class for id: "+pid);
		}
		return generateModel(cls, db);
	}

	/**
	 * Generates a new model taking as input a database D.
	 * 
	 * Note that the resulting model is uniquely identified by the modeId, which is currently constructed
	 * as a concatenation of the db and a hidden UUID state. This means that in the unlikely case that
	 * there is an existing model by this ID, it will be overwritten,
	 * 
	 * @param db
	 * @return modelId
	 * @throws OWLOntologyCreationException
	 * @throws URISyntaxException 
	 * @throws IOException 
	 */
	public String generateBlankModel(String db) throws OWLOntologyCreationException, IOException, URISyntaxException {

		// Create an arbitrary unique ID and add it to the system.
		String modelId;
		if (db != null) {
			modelId = "gomodel:" + db +"-"+ localUnique(); // TODO: another case of our temporary identifiers.
		}
		else{
			modelId = "gomodel:" + localUnique(); // TODO: another case of our temporary identifiers.
		}
		if (modelMap.containsKey(modelId)) {
			throw new OWLOntologyCreationException("A model already exists for this db: "+modelId);
		}
		LOG.info("Generating blank model for new modelId: "+modelId);

		// create empty ontology, use model id as ontology IRI
		final OWLOntologyManager m = graph.getManager();
		IRI aBoxIRI = MolecularModelJsonRenderer.getIRI(modelId, graph);
		final OWLOntology tbox = graph.getSourceOntology();
		OWLOntology abox = null;
		LegoModelGenerator model = null;
		try {
			abox = m.createOntology(aBoxIRI);
	
			// add imports to T-Box and additional ontologies via IRI
			createImports(abox, tbox.getOntologyID(), db);
			
			// generate model
			model = new LegoModelGenerator(tbox, abox);
			
			model.setPrecomputePropertyClassCombinations(isPrecomputePropertyClassCombinations);
			if (db != null) {
				GafDocument gafdoc = getGaf(db);
				model.initialize(gafdoc, graph);
				model.setContextualizingSuffix(db);
			}
		}
		catch (OWLOntologyCreationException exception) {
			if (abox != null) {
				m.removeOntology(abox);
			}
			throw exception;
		}
		catch (IOException exception) {
			if (model != null) {
				model.dispose();
			}
			throw exception;
		}
		catch (URISyntaxException exception) {
			if (model != null) {
				model.dispose();
			}
			throw exception;
		}
		
		// add to internal map
		modelMap.put(modelId, model);
		return modelId;
	}
	
	public String generateDerivedModel(String sourceModelId) throws OWLOntologyCreationException, IOException, URISyntaxException {
		LOG.info("Generating derived model from "+sourceModelId);
		LegoModelGenerator sourceModel = this.getModel(sourceModelId); 
		String modelId = this.generateBlankModel(null);
		LegoModelGenerator model = this.getModel(modelId);
		// TODO - populate, adding metadata
		Set<OWLNamedIndividual> sourceInds = this.getIndividuals(sourceModelId);
		for (OWLNamedIndividual sourceInd : sourceInds) {
			// clone sourceInd
		}
		return modelId;
	}
	
	/**
	 * Adds a process individual (and inferred individuals) to a model
	 * 
	 * @param modelId
	 * @param processCls
	 * @return null
	 * @throws OWLOntologyCreationException
	 * 
	 * @Deprecated problematic return type
	 */
	@Deprecated
	public String addProcess(String modelId, OWLClass processCls) throws OWLOntologyCreationException {
		LegoModelGenerator mod = getModel(modelId);
		Set<String> genes = new HashSet<String>();
		mod.buildNetwork(processCls, genes);
		return null;
	}

	/**
	 * 
	 * @param modelId
	 * @return all individuals in the model
	 */
	public Set<OWLNamedIndividual> getIndividuals(String modelId) {
		LegoModelGenerator mod = getModel(modelId);
		return mod.getAboxOntology().getIndividualsInSignature();
	}

	
	/**
	 * @param modelId
	 * @param q
	 * @return all individuals in the model that satisfy q
	 */
	public Set<OWLNamedIndividual> getIndividualsByQuery(String modelId, OWLClassExpression q) {
		LegoModelGenerator mod = getModel(modelId);
		return mod.getReasoner().getInstances(q, false).getFlattened();
	}

	/**
	 * @param modelId
	 * @param qs
	 * @return all individuals in the model that satisfy q
	 */
	public Set<OWLNamedIndividual> getIndividualsByQuery(String modelId, String qs) {
		LegoModelGenerator mod = getModel(modelId);
		ManchesterSyntaxTool mst = new ManchesterSyntaxTool(new OWLGraphWrapper(mod.getAboxOntology()), false);
		OWLClassExpression q = mst.parseManchesterExpression(qs);
		return getIndividualsByQuery(modelId, q);
	}

	/**
	 * Only use for testing.
	 * 
	 * @param modelId
	 * @return List of key-val pairs ready for Gson
	 */
	List<Map<Object, Object>> getIndividualObjects(String modelId) {
		LegoModelGenerator mod = getModel(modelId);
		MolecularModelJsonRenderer renderer = new MolecularModelJsonRenderer(mod.getAboxOntology());
		OWLOntology ont = mod.getAboxOntology();
		List<Map<Object, Object>> objs = new ArrayList<Map<Object, Object>>();
		for (OWLNamedIndividual i : ont.getIndividualsInSignature()) {
			objs.add(renderer.renderObject(i));
		}
		return objs;
	}
	
	/**
	 * @param modelId
	 * @return Map object ready for Gson
	 */
	public Map<Object, Object> getModelObject(String modelId) {
		LegoModelGenerator mod = getModel(modelId);
		MolecularModelJsonRenderer renderer = new MolecularModelJsonRenderer(mod.getAboxOntology());
		return renderer.renderModel();
	}

	/**
	 * Given an instance, generate the most specific class instance that classifies
	 * this instance, and add this as a class to the model ontology
	 * 
	 * @param modelId
	 * @param individualId
	 * @param newClassId
	 * @return newClassId
	 */
	public String createMostSpecificClass(String modelId, String individualId, String newClassId) {
		LegoModelGenerator mod = getModel(modelId);
		OWLIndividual ind = getIndividual(individualId, mod);
		OWLClassExpression msce = mod.getMostSpecificClassExpression((OWLNamedIndividual) ind);
		OWLClass c = getClass(newClassId, mod);
		addAxiom(mod, mod.getOWLDataFactory().getOWLEquivalentClassesAxiom(msce, c), true);
		return newClassId;
	}

	/**
	 * @param modelId
	 * @param c
	 * @return id of created individual
	 */
	public OWLOperationResponse createIndividual(String modelId, OWLClass c) {
		LegoModelGenerator model = getModel(modelId);
		OWLNamedIndividual individual = createIndividual(model, modelId, c, null, true);
		
		return createResponse(true, model, individual);
	}
	
	private OWLOperationResponse createResponse(boolean success, LegoModelGenerator model, OWLNamedIndividual...individuals) {
		OWLOperationResponse resp = new OWLOperationResponse(true, model.getReasoner().isConsistent());
		resp.setIndividuals(Arrays.asList(individuals));
		return resp;
	}
	
	/*
	 * TODO - finalize identifier policy. Currently concatenates model and class IDs
	 */
	private OWLNamedIndividual createIndividual(LegoModelGenerator model, String modelId, OWLClass c, Set<OWLAnnotation> annotations, boolean flushReasoner) {
		LOG.info("Creating individual of type: "+c);
		OWLGraphWrapper graph = new OWLGraphWrapper(model.getAboxOntology());
		String cid = graph.getIdentifier(c).replaceAll(":","-"); // e.g. GO-0123456

		// Make something unique to tag onto the generated IDs.
		String unique = localUnique();
		String iid = modelId +"-"+ cid +"-"+ unique; // TODO - unique-ify in a way that makes Chris happy
		LOG.info("  new OD: "+iid);

		IRI iri = MolecularModelJsonRenderer.getIRI(iid, graph);
		OWLDataFactory f = model.getOWLDataFactory();
		OWLNamedIndividual i = f.getOWLNamedIndividual(iri);
		
		// create axioms
		Set<OWLAxiom> axioms = new HashSet<OWLAxiom>();
		// declaration
		axioms.add(f.getOWLDeclarationAxiom(i));
		// annotation assertions
		if(annotations != null) {
			for(OWLAnnotation annotation : annotations) {
				axioms.add(f.getOWLAnnotationAssertionAxiom(iri, annotation));
			}
		}
		
		addAxioms(model, axioms, flushReasoner);
		addType(model, i, c, flushReasoner);
		return i;
	}
	
	/**
	 * Shortcut for {@link #createIndividual(String, OWLClass)}
	 * 
	 * @param modelId
	 * @param cid
	 * @param annotations
	 * @return id of created individual
	 * @throws UnknownIdentifierException 
	 */
	public OWLOperationResponse createIndividual(String modelId, String cid, Collection<Pair<String, String>> annotations) throws UnknownIdentifierException {
		LegoModelGenerator model = getModel(modelId);
		OWLClass cls = getClass(cid, model);
		if (cls == null) {
			throw new UnknownIdentifierException("Could not find a class for id: "+cid);
		}
		Set<OWLAnnotation> owlAnnotations = createAnnotations(annotations, model);
		OWLNamedIndividual individual = createIndividual(model, modelId, cls, owlAnnotations , true);
		return createResponse(true, model, individual);
	}
	
	private Set<OWLAnnotation> createAnnotations(Collection<Pair<String, String>> pairs, LegoModelGenerator model) throws UnknownIdentifierException {
		OWLDataFactory f = model.getOWLDataFactory();
		Set<OWLAnnotation> owlAnnotations = null;
		if (pairs != null && !pairs.isEmpty()) {
			owlAnnotations = new HashSet<OWLAnnotation>();
			for(Pair<String, String> pair : pairs) {
				final LegoAnnotationType type = LegoAnnotationType.getLegoType(pair.getKey());
				if (type == null) {
					throw new UnknownIdentifierException("Could not map: '"+pair.getKey()+"' to a know annotation property");
				}
				OWLAnnotationValue value;
				if (type == LegoAnnotationType.evidence) {
					String v = pair.getValue();
					OWLClass eco = getClass(v, model);
					if (eco == null) {
						throw new UnknownIdentifierException("Could not find a class for id: "+v);
					}
					value = eco.getIRI();
				}
				else {
					value = f.getOWLLiteral(pair.getValue());
				}
				OWLAnnotationProperty property = f.getOWLAnnotationProperty(type.getAnnotationProperty());
				OWLAnnotation annotation = f.getOWLAnnotation(property, value);
				owlAnnotations.add(annotation);
			}
		}
		return owlAnnotations;
	}
	
	public static final ThreadLocal<DateFormat> LegoAnnotationTypeDateFormat = new ThreadLocal<DateFormat>(){

		@Override
		protected DateFormat initialValue() {
			return new SimpleDateFormat("yyyy-MM-dd");
		}
		
	};
	
	public static enum LegoAnnotationType {
		
		comment(OWLRDFVocabulary.RDFS_COMMENT.getIRI()), // arbitrary String
		evidence(IRI.create("http://geneontology.org/lego/evidence")), // eco class iri
		date(IRI.create("http://purl.org/dc/elements/1.1/date")), // arbitrary string at the moment, define date format?
		// DC recommends http://www.w3.org/TR/NOTE-datetime, one example format is YYYY-MM-DD
		source(IRI.create("http://purl.org/dc/elements/1.1/source")), // arbitrary string, such as PMID:000000
		contributor(IRI.create("http://purl.org/dc/elements/1.1/contributor")); // who contributed to the annotation
		
		private final IRI annotationProperty;<commit_msg>add title as known meta data<commit_after>package owltools.gaf.lego;

import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.log4j.Logger;
import org.coode.owlapi.manchesterowlsyntax.ManchesterOWLSyntaxOntologyFormat;
import org.geneontology.lego.dot.LegoDotWriter;
import org.geneontology.lego.dot.LegoRenderer;
import org.geneontology.lego.model.LegoTools.UnExpectedStructureException;
import org.semanticweb.owlapi.expression.ParserException;
import org.semanticweb.owlapi.io.OWLFunctionalSyntaxOntologyFormat;
import org.semanticweb.owlapi.io.OWLOntologyDocumentSource;
import org.semanticweb.owlapi.io.OWLXMLOntologyFormat;
import org.semanticweb.owlapi.io.RDFXMLOntologyFormat;
import org.semanticweb.owlapi.io.StringDocumentSource;
import org.semanticweb.owlapi.model.AddAxiom;
import org.semanticweb.owlapi.model.AddImport;
import org.semanticweb.owlapi.model.AddOntologyAnnotation;
import org.semanticweb.owlapi.model.AxiomType;
import org.semanticweb.owlapi.model.IRI;
import org.semanticweb.owlapi.model.OWLAnnotation;
import org.semanticweb.owlapi.model.OWLAnnotationAssertionAxiom;
import org.semanticweb.owlapi.model.OWLAnnotationProperty;
import org.semanticweb.owlapi.model.OWLAnnotationValue;
import org.semanticweb.owlapi.model.OWLAxiom;
import org.semanticweb.owlapi.model.OWLAxiomChange;
import org.semanticweb.owlapi.model.OWLClass;
import org.semanticweb.owlapi.model.OWLClassAssertionAxiom;
import org.semanticweb.owlapi.model.OWLClassExpression;
import org.semanticweb.owlapi.model.OWLDataFactory;
import org.semanticweb.owlapi.model.OWLEntity;
import org.semanticweb.owlapi.model.OWLException;
import org.semanticweb.owlapi.model.OWLImportsDeclaration;
import org.semanticweb.owlapi.model.OWLIndividual;
import org.semanticweb.owlapi.model.OWLNamedIndividual;
import org.semanticweb.owlapi.model.OWLObjectProperty;
import org.semanticweb.owlapi.model.OWLObjectPropertyAssertionAxiom;
import org.semanticweb.owlapi.model.OWLObjectPropertyExpression;
import org.semanticweb.owlapi.model.OWLObjectSomeValuesFrom;
import org.semanticweb.owlapi.model.OWLOntology;
import org.semanticweb.owlapi.model.OWLOntologyAlreadyExistsException;
import org.semanticweb.owlapi.model.OWLOntologyChange;
import org.semanticweb.owlapi.model.OWLOntologyChangeVisitor;
import org.semanticweb.owlapi.model.OWLOntologyCreationException;
import org.semanticweb.owlapi.model.OWLOntologyDocumentAlreadyExistsException;
import org.semanticweb.owlapi.model.OWLOntologyFormat;
import org.semanticweb.owlapi.model.OWLOntologyID;
import org.semanticweb.owlapi.model.OWLOntologyIRIMapper;
import org.semanticweb.owlapi.model.OWLOntologyManager;
import org.semanticweb.owlapi.model.OWLOntologyStorageException;
import org.semanticweb.owlapi.model.RemoveAxiom;
import org.semanticweb.owlapi.model.RemoveImport;
import org.semanticweb.owlapi.model.RemoveOntologyAnnotation;
import org.semanticweb.owlapi.model.SetOntologyID;
import org.semanticweb.owlapi.reasoner.OWLReasoner;
import org.semanticweb.owlapi.vocab.OWLRDFVocabulary;

import owltools.gaf.GafDocument;
import owltools.gaf.bioentities.ProteinTools;
import owltools.gaf.eco.EcoMapper;
import owltools.gaf.parser.GafObjectsBuilder;
import owltools.graph.OWLGraphWrapper;
import owltools.io.CatalogXmlIRIMapper;
import owltools.vocab.OBOUpperVocabulary;

/**
 * Manager object for operations on collections of MolecularModels (aka lego diagrams)
 * 
 * any number of models can be loaded at any time (todo - impose some limit to avoid
 * using too much memory)
 * 
 * each model has a generator, an OWLOntology (containing the set of class assertions)
 * and a reasoner associated with it (todo - test memory requirements)
 * 
 * This manager is designed to be used within a web server. Multiple clients can
 * contact the same manager instance through services
 *
 */
public class MolecularModelManager {
	
	private static Logger LOG = Logger.getLogger(MolecularModelManager.class);


	OWLGraphWrapper graph;
	boolean isPrecomputePropertyClassCombinations = false;
	Map<String, GafDocument> dbToGafdoc = new HashMap<String, GafDocument>();
	Map<String, LegoModelGenerator> modelMap = new HashMap<String, LegoModelGenerator>();
	String pathToGafs = "gene-associations";
	String pathToOWLFiles = "owl-models";
	String pathToProteinFiles = null;
	Map<String, String> dbToTaxon = null;
	OWLOntologyIRIMapper proteinMapper = null;
	
	GafObjectsBuilder builder = new GafObjectsBuilder();
	// WARNING: Do *NOT* switch to functional syntax until the OWL-API has fixed a bug.
	OWLOntologyFormat ontologyFormat = new ManchesterOWLSyntaxOntologyFormat();
	Set<IRI> additionalImports;
	final Set<IRI> obsoleteImports = new HashSet<IRI>();

	// TODO: Temporarily for keeping instances unique (search for "unique" below).
	static String uniqueTop = Long.toHexString((System.currentTimeMillis()/1000));
	static long instanceCounter = 0;
	private static String localUnique(){
		instanceCounter++;
		String unique = uniqueTop + String.format("%07d", instanceCounter);
		return unique;		
	}

	/**
	 * Represents the reponse to a requested translation on an
	 * ontology/model
	 * 
	 */
	public class OWLOperationResponse {
		OWLAxiomChange change;
		int changeId;
		boolean isSuccess = true;
		boolean isResultsInInconsistency = false;
		Object modelData = null;
		List<OWLNamedIndividual> individuals = null;
		
		/**
		 * @param isSuccess
		 */
		public OWLOperationResponse(boolean isSuccess) {
			super();
			this.isSuccess = isSuccess;
		}
		
		/**
		 * @param isSuccess
		 * @param isResultsInInconsistency
		 */
		public OWLOperationResponse(boolean isSuccess,
				boolean isResultsInInconsistency) {
			super();
			this.isSuccess = isSuccess;
			this.isResultsInInconsistency = isResultsInInconsistency;
		}
		
		public OWLOperationResponse(OWLAxiomChange change, boolean isSuccess,
				boolean isResultsInInconsistency) {
			super();
			this.isSuccess = isSuccess;
			this.isResultsInInconsistency = isResultsInInconsistency;
			this.change = change;
		}
		
		/**
		 * @return the isSuccess
		 */
		public boolean isSuccess() {
			return isSuccess;
		}
		
		/**
		 * @return the isResultsInInconsistency
		 */
		public boolean isResultsInInconsistency() {
			return isResultsInInconsistency;
		}
		
		/**
		 * @return the modelData
		 */
		public Object getModelData() {
			return modelData;
		}
		
		/**
		 * @param modelData the modelData to set
		 */
		public void setModelData(Object modelData) {
			this.modelData = modelData;
		}
		
		/**
		 * @return the individuals
		 */
		public List<OWLNamedIndividual> getIndividuals() {
			return individuals;
		}

		/**
		 * @param individuals the individuals to set
		 */
		public void setIndividuals(List<OWLNamedIndividual> individuals) {
			this.individuals = individuals;
		}
		
		
//		/**
//		 * Combine two responses together.
//		 * 
//		 * @param response
//		 */
//		public void mergeIn(OWLOperationResponse toMerge) {
//
//			// success and inconsistency dominate false and true respectively.
//			if( ! toMerge.isSuccess ) this.isSuccess = false;
//			if( ! toMerge.isResultsInInconsistency() ) this.isResultsInInconsistency = true;
//			
//			// Merge model data.
//			Map<Object, Object> newMap = new HashMap<Object,Object>();
//			List<Map<Object, Object>> toLoop = this.getModelData(); // add this response
//			toLoop.addAll(toMerge.getModelData()); // add merging response
//			for( Map<Object, Object>mergable : toLoop){
//				// If new, add to new.
//				if( mergable.containsKey("id") ){
//					
//				}
//			}
//			
//			// Merge ID list via Set (no dupes).
//		    Set<String> set = new HashSet<String>();
//		    set.addAll(this.getIndividualIds());
//		    set.addAll(toMerge.getIndividualIds());
//			this.individualIds = new ArrayList<String>(set);
//		}
		
	}

	public static class UnknownIdentifierException extends Exception {

		// generated
		private static final long serialVersionUID = -847970910712518838L;

		/**
		 * @param message
		 * @param cause
		 */
		public UnknownIdentifierException(String message, Throwable cause) {
			super(message, cause);
		}

		/**
		 * @param message
		 */
		public UnknownIdentifierException(String message) {
			super(message);
		}

	}
	
	/**
	 * @param graph
	 * @throws OWLOntologyCreationException
	 */
	public MolecularModelManager(OWLGraphWrapper graph) throws OWLOntologyCreationException {
		super();
		this.graph = graph;
		init();
	}
	/**
	 * @param ont
	 * @throws OWLOntologyCreationException
	 */
	public MolecularModelManager(OWLOntology ont) throws OWLOntologyCreationException {
		super();
		this.graph = new OWLGraphWrapper(ont);
		init();
	}

	/**
	 * @throws OWLOntologyCreationException
	 */
	protected void init() throws OWLOntologyCreationException {
		// set default imports
		additionalImports = new HashSet<IRI>();
		additionalImports.add(IRI.create("http://purl.obolibrary.org/obo/ro.owl"));
		additionalImports.add(IRI.create("http://purl.obolibrary.org/obo/go/extensions/ro_pending.owl"));
		additionalImports.add(EcoMapper.ECO_PURL_IRI);
	}


	/**
	 * @return graph wrapper for core/source ontology
	 */
	public OWLGraphWrapper getGraph() {
		return graph;
	}

	/**
	 * @return core/source ontology
	 */
	public OWLOntology getOntology() {
		return graph.getSourceOntology();
	}


	/**
	 * @return path to gafs direcory
	 */
	public String getPathToGafs() {
		return pathToGafs;
	}
	/**
	 * Can either be an HTTP prefix, or an absolute file path
	 * 
	 * @param pathToGafs
	 */
	public void setPathToGafs(String pathToGafs) {
		this.pathToGafs = pathToGafs;
	}
	
	/**
	 * Note this may move to an implementation-specific subclass in future
	 * 
	 * @return path to owl on server
	 */
	public String getPathToOWLFiles() {
		return pathToOWLFiles;
	}
	/**
	 * @param pathToOWLFiles
	 */
	public void setPathToOWLFiles(String pathToOWLFiles) {
		this.pathToOWLFiles = pathToOWLFiles;
	}
	
	/**
	 * @param pathToProteinFiles
	 * @throws IOException 
	 */
	public synchronized void setPathToProteinFiles(String pathToProteinFiles) throws IOException {
		setPathToProteinFiles(pathToProteinFiles, "catalog-v001.xml");
	}
	
	/**
	 * @param pathToProteinFiles
	 * @param catalogXML
	 * @throws IOException 
	 */
	public synchronized void setPathToProteinFiles(String pathToProteinFiles, String catalogXML) throws IOException {
		if (proteinMapper != null) {
			graph.getManager().removeIRIMapper(proteinMapper);
		}
		this.pathToProteinFiles = pathToProteinFiles;
		proteinMapper = new CatalogXmlIRIMapper(new File(pathToProteinFiles, catalogXML));
		graph.getManager().addIRIMapper(proteinMapper);
	}
	
	public String getPathToProteinFiles() {
		return pathToProteinFiles;
	}
	
	/**
	 * @return the dbToTaxon
	 */
	public Map<String, String> getDbToTaxon() {
		return dbToTaxon;
	}
	/**
	 * @param dbToTaxon the dbToTaxon to set
	 */
	public void setDbToTaxon(Map<String, String> dbToTaxon) {
		this.dbToTaxon = dbToTaxon;
	}
	/**
	 * loads/register a Gaf document
	 * 
	 * @param db
	 * @return Gaf document
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public GafDocument loadGaf(String db) throws IOException, URISyntaxException {
		if (!dbToGafdoc.containsKey(db)) {
			LOG.info("Loading GAF for db: "+db);
			GafDocument gafdoc = builder.buildDocument(pathToGafs + "/gene_association." + db + ".gz");
			dbToGafdoc.put(db, gafdoc);
		}
		return dbToGafdoc.get(db);
	}

	/**
	 * Loads and caches a GAF document from a specified location
	 * 
	 * @param db
	 * @param gafFile
	 * @return Gaf document
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public GafDocument loadGaf(String db, File gafFile) throws IOException, URISyntaxException {
		if (!dbToGafdoc.containsKey(db)) {

			GafDocument gafdoc = builder.buildDocument(gafFile);
			dbToGafdoc.put(db, gafdoc);
		}
		return dbToGafdoc.get(db);
	}


	/**
	 * @param db
	 * @return Gaf document for db
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public GafDocument getGaf(String db) throws IOException, URISyntaxException {
		return loadGaf(db);
	}

	/**
	 * Add additional import declarations for any newly generated model.
	 * 
	 * @param imports
	 */
	public void addImports(Iterable<String> imports) {
		if (imports != null) {
			for (String importIRIString : imports) {
				additionalImports.add(IRI.create(importIRIString));
			}
		}
	}
	
	/**
	 * Mark the given imports as obsolete.
	 * 
	 * @param obsoletes
	 */
	public void addObsoleteImports(Iterable<String> obsoletes) {
		if (obsoletes != null) {
			for (String obsolete : obsoletes) {
				obsoleteImports.add(IRI.create(obsolete));
			}
		}
	}
	
	public Collection<IRI> getImports() {
		return Collections.unmodifiableCollection(additionalImports);
	}

	/**
	 * Generates a new model taking as input a biological process P and a database D.
	 * The axioms from P and annotations to P from D are used to seed a new model
	 * 
	 * See {@link LegoModelGenerator#buildNetwork(OWLClass, java.util.Collection)}
	 * And also https://docs.google.com/document/d/1TV8Eb9sSvFY-weVZaIfzCxF1qbnmkUaiUhTm9Bs3gRE/edit
	 * 
	 * Note the resulting model is uniquely identified by the modeId, which is currently constructed
	 * as a concatenation of the db and the P id. This means that if there is an existing model by
	 * this ID it will be overwritten
	 * 
	 * @param processCls
	 * @param db
	 * @return modelId
	 * @throws OWLOntologyCreationException
	 * @throws URISyntaxException 
	 * @throws IOException 
	 */
	public String generateModel(OWLClass processCls, String db) throws OWLOntologyCreationException, IOException, URISyntaxException {
		// quick check, only generate a model if it does not already exists.
		final String p = graph.getIdentifier(processCls);
		String modelId = getModelId(p, db);
		if (modelMap.containsKey(modelId)) {
			throw new OWLOntologyCreationException("A model already exists for this process and db: "+modelId);
		}
		LOG.info("Generating model for Class: "+p+" and db: "+db);
		OWLOntology tbox = graph.getSourceOntology();
		OWLOntology abox = null;
		LegoModelGenerator model = null;
		
		// create empty ontology
		// use model id as ontology IRI
		OWLOntologyManager m = graph.getManager();
		IRI iri = MolecularModelJsonRenderer.getIRI(modelId, graph);
		try {
			abox = m.createOntology(iri);
			
			// setup model ontology to import the source ontology and other imports
			createImports(abox, tbox.getOntologyID(), db);
			
			// create generator
			model = new LegoModelGenerator(tbox, abox);
			
			model.setPrecomputePropertyClassCombinations(isPrecomputePropertyClassCombinations);
			Set<String> seedGenes = new HashSet<String>();
			// only look for genes if a GAF is available
			if (db != null) {
				GafDocument gafdoc = getGaf(db);
				model.initialize(gafdoc, graph);
				seedGenes.addAll(model.getGenes(processCls));
			}
			model.setContextualizingSuffix(db);
			model.buildNetwork(p, seedGenes);
		}
		catch (OWLOntologyCreationException exception) {
			if (model != null) {
				model.dispose();
			} else if (abox != null) {
				m.removeOntology(abox);
			}
			throw exception;
		}
		catch (IOException exception) {
			if (model != null) {
				model.dispose();
			}
			throw exception;
		}
		catch (URISyntaxException exception) {
			if (model != null) {
				model.dispose();
			}
			throw exception;
		}
		modelMap.put(modelId, model);
		return modelId;

	}

	private void createImports(OWLOntology ont, OWLOntologyID tboxId, String db) throws OWLOntologyCreationException {
		OWLOntologyManager m = ont.getOWLOntologyManager();
		OWLDataFactory f = m.getOWLDataFactory();
		
		// import T-Box
		OWLImportsDeclaration tBoxImportDeclaration = f.getOWLImportsDeclaration(tboxId.getOntologyIRI());
		m.applyChange(new AddImport(ont, tBoxImportDeclaration));
		
		// import additional ontologies via IRI
		for (IRI importIRI : additionalImports) {
			OWLImportsDeclaration importDeclaration = f.getOWLImportsDeclaration(importIRI);
			// check that the import ontology is available
			OWLOntology importOntology = m.getOntology(importIRI);
			if (importOntology == null) {
				// only try to load it, if it isn't already loaded
				try {
					m.loadOntology(importIRI);
				} catch (OWLOntologyDocumentAlreadyExistsException e) {
					// ignore
				} catch (OWLOntologyAlreadyExistsException e) {
					// ignore
				}
			}
			m.applyChange(new AddImport(ont, importDeclaration));
		}
		
		// check for protein ontology
		if (db != null && pathToProteinFiles != null && proteinMapper != null) {
			String taxonId = null;
			if (dbToTaxon != null) {
				taxonId = dbToTaxon.get(db);
			}
			if (taxonId == null) {
				// fallback
				taxonId = db;
			}
			IRI proteinIRI = ProteinTools.createProteinOntologyIRI(taxonId);
			IRI mapped = proteinMapper.getDocumentIRI(proteinIRI);
			if (mapped != null) {
				OWLImportsDeclaration importDeclaration = f.getOWLImportsDeclaration(proteinIRI);
				m.loadOntology(proteinIRI);
				m.applyChange(new AddImport(ont, importDeclaration));
			}
		}
	}
	
	/**
	 * wrapper for {@link #generateModel(OWLClass, String)}
	 * 
	 * @param pid
	 * @param db
	 * @return modelId
	 * @throws OWLOntologyCreationException
	 * @throws IOException
	 * @throws URISyntaxException
	 * @throws UnknownIdentifierException
	 */
	public String generateModel(String pid, String db) throws OWLOntologyCreationException, IOException, URISyntaxException, UnknownIdentifierException {
		OWLClass cls = getClass(pid, graph);
		if (cls == null) {
			throw new UnknownIdentifierException("Could not find a class for id: "+pid);
		}
		return generateModel(cls, db);
	}

	/**
	 * Generates a new model taking as input a database D.
	 * 
	 * Note that the resulting model is uniquely identified by the modeId, which is currently constructed
	 * as a concatenation of the db and a hidden UUID state. This means that in the unlikely case that
	 * there is an existing model by this ID, it will be overwritten,
	 * 
	 * @param db
	 * @return modelId
	 * @throws OWLOntologyCreationException
	 * @throws URISyntaxException 
	 * @throws IOException 
	 */
	public String generateBlankModel(String db) throws OWLOntologyCreationException, IOException, URISyntaxException {

		// Create an arbitrary unique ID and add it to the system.
		String modelId;
		if (db != null) {
			modelId = "gomodel:" + db +"-"+ localUnique(); // TODO: another case of our temporary identifiers.
		}
		else{
			modelId = "gomodel:" + localUnique(); // TODO: another case of our temporary identifiers.
		}
		if (modelMap.containsKey(modelId)) {
			throw new OWLOntologyCreationException("A model already exists for this db: "+modelId);
		}
		LOG.info("Generating blank model for new modelId: "+modelId);

		// create empty ontology, use model id as ontology IRI
		final OWLOntologyManager m = graph.getManager();
		IRI aBoxIRI = MolecularModelJsonRenderer.getIRI(modelId, graph);
		final OWLOntology tbox = graph.getSourceOntology();
		OWLOntology abox = null;
		LegoModelGenerator model = null;
		try {
			abox = m.createOntology(aBoxIRI);
	
			// add imports to T-Box and additional ontologies via IRI
			createImports(abox, tbox.getOntologyID(), db);
			
			// generate model
			model = new LegoModelGenerator(tbox, abox);
			
			model.setPrecomputePropertyClassCombinations(isPrecomputePropertyClassCombinations);
			if (db != null) {
				GafDocument gafdoc = getGaf(db);
				model.initialize(gafdoc, graph);
				model.setContextualizingSuffix(db);
			}
		}
		catch (OWLOntologyCreationException exception) {
			if (abox != null) {
				m.removeOntology(abox);
			}
			throw exception;
		}
		catch (IOException exception) {
			if (model != null) {
				model.dispose();
			}
			throw exception;
		}
		catch (URISyntaxException exception) {
			if (model != null) {
				model.dispose();
			}
			throw exception;
		}
		
		// add to internal map
		modelMap.put(modelId, model);
		return modelId;
	}
	
	public String generateDerivedModel(String sourceModelId) throws OWLOntologyCreationException, IOException, URISyntaxException {
		LOG.info("Generating derived model from "+sourceModelId);
		LegoModelGenerator sourceModel = this.getModel(sourceModelId); 
		String modelId = this.generateBlankModel(null);
		LegoModelGenerator model = this.getModel(modelId);
		// TODO - populate, adding metadata
		Set<OWLNamedIndividual> sourceInds = this.getIndividuals(sourceModelId);
		for (OWLNamedIndividual sourceInd : sourceInds) {
			// clone sourceInd
		}
		return modelId;
	}
	
	/**
	 * Adds a process individual (and inferred individuals) to a model
	 * 
	 * @param modelId
	 * @param processCls
	 * @return null
	 * @throws OWLOntologyCreationException
	 * 
	 * @Deprecated problematic return type
	 */
	@Deprecated
	public String addProcess(String modelId, OWLClass processCls) throws OWLOntologyCreationException {
		LegoModelGenerator mod = getModel(modelId);
		Set<String> genes = new HashSet<String>();
		mod.buildNetwork(processCls, genes);
		return null;
	}

	/**
	 * 
	 * @param modelId
	 * @return all individuals in the model
	 */
	public Set<OWLNamedIndividual> getIndividuals(String modelId) {
		LegoModelGenerator mod = getModel(modelId);
		return mod.getAboxOntology().getIndividualsInSignature();
	}

	
	/**
	 * @param modelId
	 * @param q
	 * @return all individuals in the model that satisfy q
	 */
	public Set<OWLNamedIndividual> getIndividualsByQuery(String modelId, OWLClassExpression q) {
		LegoModelGenerator mod = getModel(modelId);
		return mod.getReasoner().getInstances(q, false).getFlattened();
	}

	/**
	 * @param modelId
	 * @param qs
	 * @return all individuals in the model that satisfy q
	 */
	public Set<OWLNamedIndividual> getIndividualsByQuery(String modelId, String qs) {
		LegoModelGenerator mod = getModel(modelId);
		ManchesterSyntaxTool mst = new ManchesterSyntaxTool(new OWLGraphWrapper(mod.getAboxOntology()), false);
		OWLClassExpression q = mst.parseManchesterExpression(qs);
		return getIndividualsByQuery(modelId, q);
	}

	/**
	 * Only use for testing.
	 * 
	 * @param modelId
	 * @return List of key-val pairs ready for Gson
	 */
	List<Map<Object, Object>> getIndividualObjects(String modelId) {
		LegoModelGenerator mod = getModel(modelId);
		MolecularModelJsonRenderer renderer = new MolecularModelJsonRenderer(mod.getAboxOntology());
		OWLOntology ont = mod.getAboxOntology();
		List<Map<Object, Object>> objs = new ArrayList<Map<Object, Object>>();
		for (OWLNamedIndividual i : ont.getIndividualsInSignature()) {
			objs.add(renderer.renderObject(i));
		}
		return objs;
	}
	
	/**
	 * @param modelId
	 * @return Map object ready for Gson
	 */
	public Map<Object, Object> getModelObject(String modelId) {
		LegoModelGenerator mod = getModel(modelId);
		MolecularModelJsonRenderer renderer = new MolecularModelJsonRenderer(mod.getAboxOntology());
		return renderer.renderModel();
	}

	/**
	 * Given an instance, generate the most specific class instance that classifies
	 * this instance, and add this as a class to the model ontology
	 * 
	 * @param modelId
	 * @param individualId
	 * @param newClassId
	 * @return newClassId
	 */
	public String createMostSpecificClass(String modelId, String individualId, String newClassId) {
		LegoModelGenerator mod = getModel(modelId);
		OWLIndividual ind = getIndividual(individualId, mod);
		OWLClassExpression msce = mod.getMostSpecificClassExpression((OWLNamedIndividual) ind);
		OWLClass c = getClass(newClassId, mod);
		addAxiom(mod, mod.getOWLDataFactory().getOWLEquivalentClassesAxiom(msce, c), true);
		return newClassId;
	}

	/**
	 * @param modelId
	 * @param c
	 * @return id of created individual
	 */
	public OWLOperationResponse createIndividual(String modelId, OWLClass c) {
		LegoModelGenerator model = getModel(modelId);
		OWLNamedIndividual individual = createIndividual(model, modelId, c, null, true);
		
		return createResponse(true, model, individual);
	}
	
	private OWLOperationResponse createResponse(boolean success, LegoModelGenerator model, OWLNamedIndividual...individuals) {
		OWLOperationResponse resp = new OWLOperationResponse(true, model.getReasoner().isConsistent());
		resp.setIndividuals(Arrays.asList(individuals));
		return resp;
	}
	
	/*
	 * TODO - finalize identifier policy. Currently concatenates model and class IDs
	 */
	private OWLNamedIndividual createIndividual(LegoModelGenerator model, String modelId, OWLClass c, Set<OWLAnnotation> annotations, boolean flushReasoner) {
		LOG.info("Creating individual of type: "+c);
		OWLGraphWrapper graph = new OWLGraphWrapper(model.getAboxOntology());
		String cid = graph.getIdentifier(c).replaceAll(":","-"); // e.g. GO-0123456

		// Make something unique to tag onto the generated IDs.
		String unique = localUnique();
		String iid = modelId +"-"+ cid +"-"+ unique; // TODO - unique-ify in a way that makes Chris happy
		LOG.info("  new OD: "+iid);

		IRI iri = MolecularModelJsonRenderer.getIRI(iid, graph);
		OWLDataFactory f = model.getOWLDataFactory();
		OWLNamedIndividual i = f.getOWLNamedIndividual(iri);
		
		// create axioms
		Set<OWLAxiom> axioms = new HashSet<OWLAxiom>();
		// declaration
		axioms.add(f.getOWLDeclarationAxiom(i));
		// annotation assertions
		if(annotations != null) {
			for(OWLAnnotation annotation : annotations) {
				axioms.add(f.getOWLAnnotationAssertionAxiom(iri, annotation));
			}
		}
		
		addAxioms(model, axioms, flushReasoner);
		addType(model, i, c, flushReasoner);
		return i;
	}
	
	/**
	 * Shortcut for {@link #createIndividual(String, OWLClass)}
	 * 
	 * @param modelId
	 * @param cid
	 * @param annotations
	 * @return id of created individual
	 * @throws UnknownIdentifierException 
	 */
	public OWLOperationResponse createIndividual(String modelId, String cid, Collection<Pair<String, String>> annotations) throws UnknownIdentifierException {
		LegoModelGenerator model = getModel(modelId);
		OWLClass cls = getClass(cid, model);
		if (cls == null) {
			throw new UnknownIdentifierException("Could not find a class for id: "+cid);
		}
		Set<OWLAnnotation> owlAnnotations = createAnnotations(annotations, model);
		OWLNamedIndividual individual = createIndividual(model, modelId, cls, owlAnnotations , true);
		return createResponse(true, model, individual);
	}
	
	private Set<OWLAnnotation> createAnnotations(Collection<Pair<String, String>> pairs, LegoModelGenerator model) throws UnknownIdentifierException {
		OWLDataFactory f = model.getOWLDataFactory();
		Set<OWLAnnotation> owlAnnotations = null;
		if (pairs != null && !pairs.isEmpty()) {
			owlAnnotations = new HashSet<OWLAnnotation>();
			for(Pair<String, String> pair : pairs) {
				final LegoAnnotationType type = LegoAnnotationType.getLegoType(pair.getKey());
				if (type == null) {
					throw new UnknownIdentifierException("Could not map: '"+pair.getKey()+"' to a know annotation property");
				}
				OWLAnnotationValue value;
				if (type == LegoAnnotationType.evidence) {
					String v = pair.getValue();
					OWLClass eco = getClass(v, model);
					if (eco == null) {
						throw new UnknownIdentifierException("Could not find a class for id: "+v);
					}
					value = eco.getIRI();
				}
				else {
					value = f.getOWLLiteral(pair.getValue());
				}
				OWLAnnotationProperty property = f.getOWLAnnotationProperty(type.getAnnotationProperty());
				OWLAnnotation annotation = f.getOWLAnnotation(property, value);
				owlAnnotations.add(annotation);
			}
		}
		return owlAnnotations;
	}
	
	public static final ThreadLocal<DateFormat> LegoAnnotationTypeDateFormat = new ThreadLocal<DateFormat>(){

		@Override
		protected DateFormat initialValue() {
			return new SimpleDateFormat("yyyy-MM-dd");
		}
		
	};
	
	public static enum LegoAnnotationType {
		
		comment(OWLRDFVocabulary.RDFS_COMMENT.getIRI()), // arbitrary String
		evidence(IRI.create("http://geneontology.org/lego/evidence")), // eco class iri
		date(IRI.create("http://purl.org/dc/elements/1.1/date")), // arbitrary string at the moment, define date format?
		// DC recommends http://www.w3.org/TR/NOTE-datetime, one example format is YYYY-MM-DD
		source(IRI.create("http://purl.org/dc/elements/1.1/source")), // arbitrary string, such as PMID:000000
		contributor(IRI.create("http://purl.org/dc/elements/1.1/contributor")), // who contributed to the annotation
		title(IRI.create("http://purl.org/dc/elements/1.1/title")); // title (of the model)
		
		private final IRI annotationProperty;
{'fraction_non_alphanumeric': 0.1033302801444792, 'fraction_numerical': 0.0019701709209684944, 'mean_word_length': 4.319849223131378, 'pattern_counts': {'":': 2, '<': 111, '<?xml version=': 0, '>': 111, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1795993', 'n_tokens_mistral': 20675, 'n_tokens_neox': 17948, 'n_words': 4339}