<commit_before>package uk.ac.ebi.phis.xmlDump;

import java.io.File;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Map;

import javax.sql.DataSource;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Marshaller;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;

import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.w3c.dom.DOMException;

import uk.ac.ebi.phis.jaxb.Age;
import uk.ac.ebi.phis.jaxb.Annotation;
import uk.ac.ebi.phis.jaxb.AnnotationArray;
import uk.ac.ebi.phis.jaxb.AnnotationMode;
import uk.ac.ebi.phis.jaxb.Channel;
import uk.ac.ebi.phis.jaxb.Coordinates;
import uk.ac.ebi.phis.jaxb.Dimensions;
import uk.ac.ebi.phis.jaxb.Doc;
import uk.ac.ebi.phis.jaxb.Genotype;
import uk.ac.ebi.phis.jaxb.GenotypeComponent;
import uk.ac.ebi.phis.jaxb.Image;
import uk.ac.ebi.phis.jaxb.ImageDescription;
import uk.ac.ebi.phis.jaxb.ImageType;
import uk.ac.ebi.phis.jaxb.ImageTypeArray;
import uk.ac.ebi.phis.jaxb.Link;
import uk.ac.ebi.phis.jaxb.OntologyTerm;
import uk.ac.ebi.phis.jaxb.OntologyTermArray;
import uk.ac.ebi.phis.jaxb.Organism;
import uk.ac.ebi.phis.jaxb.PercentArray;
import uk.ac.ebi.phis.jaxb.Roi;
import uk.ac.ebi.phis.jaxb.SampleType;
import uk.ac.ebi.phis.jaxb.Sex;
import uk.ac.ebi.phis.jaxb.StringArray;
import uk.ac.ebi.phis.jaxb.Zygosity;
import uk.ac.ebi.phis.utils.EnrichingUtils;
import uk.ac.ebi.phis.utils.Normalizer;

public class SangerXmlGenerator {
	
	Normalizer norm;
		
	public SangerXmlGenerator(){
		norm = new Normalizer();
	}
	
	public void exportImages() throws IOException{
		
        ApplicationContext ac = new ClassPathXmlApplicationContext("app-config.xml");
		DataSource dataSource = (DataSource) ac.getBean("komp2DataSource");
        
        String command = "SELECT iir.ID, iir.LARGE_THUMBNAIL_FILE_PATH, iir.PUBLISHED_STATUS_ID, " +
        					"iit.TAG_NAME, iit.TAG_VALUE, iit.X_START, iit.X_END, iit.Y_START, iit.Y_END, " +
        					"aa.TERM_ID, aa.TERM_NAME, aa.ONTOLOGY_DICT_ID, " +
        					"imam.MOUSE_ID, imam.MOUSE_NAME, imam.GENDER, imam.AGE_IN_WEEKS, imam.GENE, imam.ALLELE, imam.GENOTYPE, " +
        					"ied.NAME as procedure_name, " +
        					"allele.symbol, allele.name, allele.acc, allele.gf_acc " +
        				"FROM IMA_IMAGE_RECORD iir " +
        					"LEFT OUTER JOIN IMA_IMAGE_TAG iit ON iit.IMAGE_RECORD_ID=iir.ID " +
        					"LEFT OUTER JOIN ANN_ANNOTATION aa ON aa.FOREIGN_KEY_ID=iit.ID " +
        					"LEFT OUTER JOIN IMPC_MOUSE_ALLELE_MV imam on imam.MOUSE_ID=iir.FOREIGN_KEY_ID " +
        					"LEFT OUTER JOIN IMA_SUBCONTEXT isub ON iir.subcontext_id=isub.id " +
        					"LEFT OUTER JOIN IMA_EXPERIMENT_DICT ied ON isub.experiment_dict_id=ied.id " +
        					"LEFT OUTER JOIN allele ON allele.symbol=imam.ALLELE " + 
        				"WHERE ied.NAME != 'Mouse Necropsy' " +
        				"ORDER BY iir.ID;";
        
        try {
        	
        	PreparedStatement statement = dataSource.getConnection().prepareStatement(command);
     		ResultSet res = statement.executeQuery();
                  	
			Doc doc = new Doc();			
			int i = 0;
			
	        while ( res.next()){

	        	if (res.getString("gf_acc") != null && res.getString("gf_acc").startsWith("MGI:")){
    				
		        	boolean sameImage = true;
		        	
		        	String internalId =  "komp2_" + res.getString("ID");
		        	
		        	String imageId = res.getString("ID");		        
		    		
		    		String url = "http://www.mousephenotype.org/data/media/" + res.getString("LARGE_THUMBNAIL_FILE_PATH") ;
		    		Map<String, Integer> dimensions = EnrichingUtils.getImageMeasuresFromUrl(url);
		    		
		    		if (dimensions != null){// the image could be loaded 	    		

			    		Boolean expression = false;
			    		Boolean phenotypeAnatomy = false;
		    			Boolean isMutant = true;
			    		
		    			Image image = new Image();
			    		image.setId(internalId);
			    		
			    		Dimensions d = new Dimensions();
		    			d.setImageHeight(dimensions.get("height"));
		    			d.setImageWidth(dimensions.get("width"));
			    		String procedure = res.getString("procedure_name");
			    		// store procedure name as observation
			    		image.setObservations(new StringArray());
			    		image.getObservations().getEl().add("Procedure name: " + procedure);

			    		Organism organism = new Organism();
			    		Sex sex = Sex.fromValue(norm.normalizeSex(res.getString("GENDER")));
			    		organism.setSex(sex);
			    		organism.setOrganismId(res.getString("MOUSE_NAME"));
			    		Age age = new Age();
			    		if (ageIsRelevant(procedure)){
			    			if (norm.isEmbryonicAge(res.getString("AGE_IN_WEEKS"))){
				    			age.setEmbryonicAge(norm.getAgeInDays(res.getString("AGE_IN_WEEKS")));
				    		} else {
				    			age.setAgeSinceBirth(norm.getAgeInDays(res.getString("AGE_IN_WEEKS")));
				    		}
				    		organism.setAge(age);
			    		}
			    		OntologyTerm stageOt = getStageFromProcedureOrAge(procedure, res.getString("AGE_IN_WEEKS"));
			    		if (stageOt != null){
			    			organism.setStage(stageOt);
			    		}
			    			
			    		organism.setTaxon("Mus musculus");
			    		image.setOrganism(organism);
			    		
			    		if (res.getString("GENOTYPE").equalsIgnoreCase("WT")){
			    			isMutant = false;
			    		}
			    		
			    		GenotypeComponent gt = new GenotypeComponent();
			    		gt.setGeneId(res.getString("gf_acc"));
			    		gt.setGeneSymbol(res.getString("GENE"));
			    		gt.setGeneticFeatureId(res.getString("acc"));
			    		gt.setGeneticFeatureSymbol(res.getString("ALLELE"));
			    		Zygosity zyg = Zygosity.fromValue(norm.normalizeZygosity(res.getString("GENOTYPE")));
			    		gt.setZygosity(zyg);
			    		Genotype gta = new Genotype();
			    		gta.getEl().add(gt);
			    		image.setMutantGenotypeTraits(gta);
			    		
			    		
				        
				        /* 	Channel 	*/
			    		String imageType = norm.getImageType(res.getString("procedure_name"));
			    		Channel channel = null;
			    		String channelId = "";
			    		if (imageType.equalsIgnoreCase("expression")){
			    			expression = true;
			    			channelId = internalId.replace("komp2_", "komp2_channel_") + "_" + 0; // we know that for Sanger data there is at most one channel.
			    			channel = new Channel();
					        channel.setAssociatedImage(internalId);
					        // KOMP2 always has at most 1 channel
					        StringArray c = new StringArray();
					        c.getEl().add(channelId);
					        image.setAssociatedChannel(c);
				    		channel.setId(channelId);
				    		channel.setDepictsExpressionOf(gt);
		    			}
			    			    			

				        /*	ROI	*/
			    		int k = 0;
			    		// Go through all annotations for the same image<commit_msg>Fixed bug with exporting genomic information for WTSI WT samples. <commit_after>package uk.ac.ebi.phis.xmlDump;

import java.io.File;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Map;

import javax.sql.DataSource;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Marshaller;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;

import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.w3c.dom.DOMException;

import uk.ac.ebi.phis.jaxb.Age;
import uk.ac.ebi.phis.jaxb.Annotation;
import uk.ac.ebi.phis.jaxb.AnnotationArray;
import uk.ac.ebi.phis.jaxb.AnnotationMode;
import uk.ac.ebi.phis.jaxb.Channel;
import uk.ac.ebi.phis.jaxb.Coordinates;
import uk.ac.ebi.phis.jaxb.Dimensions;
import uk.ac.ebi.phis.jaxb.Doc;
import uk.ac.ebi.phis.jaxb.Genotype;
import uk.ac.ebi.phis.jaxb.GenotypeComponent;
import uk.ac.ebi.phis.jaxb.Image;
import uk.ac.ebi.phis.jaxb.ImageDescription;
import uk.ac.ebi.phis.jaxb.ImageType;
import uk.ac.ebi.phis.jaxb.ImageTypeArray;
import uk.ac.ebi.phis.jaxb.Link;
import uk.ac.ebi.phis.jaxb.OntologyTerm;
import uk.ac.ebi.phis.jaxb.OntologyTermArray;
import uk.ac.ebi.phis.jaxb.Organism;
import uk.ac.ebi.phis.jaxb.PercentArray;
import uk.ac.ebi.phis.jaxb.Roi;
import uk.ac.ebi.phis.jaxb.SampleType;
import uk.ac.ebi.phis.jaxb.Sex;
import uk.ac.ebi.phis.jaxb.StringArray;
import uk.ac.ebi.phis.jaxb.Zygosity;
import uk.ac.ebi.phis.utils.EnrichingUtils;
import uk.ac.ebi.phis.utils.Normalizer;

public class SangerXmlGenerator {
	
	Normalizer norm;
		
	public SangerXmlGenerator(){
		norm = new Normalizer();
	}
	
	public void exportImages() throws IOException{
		
        ApplicationContext ac = new ClassPathXmlApplicationContext("app-config.xml");
		DataSource dataSource = (DataSource) ac.getBean("komp2DataSource");
        
        String command = "SELECT iir.ID, iir.LARGE_THUMBNAIL_FILE_PATH, iir.PUBLISHED_STATUS_ID, " +
        					"iit.TAG_NAME, iit.TAG_VALUE, iit.X_START, iit.X_END, iit.Y_START, iit.Y_END, " +
        					"aa.TERM_ID, aa.TERM_NAME, aa.ONTOLOGY_DICT_ID, " +
        					"imam.MOUSE_ID, imam.MOUSE_NAME, imam.GENDER, imam.AGE_IN_WEEKS, imam.GENE, imam.ALLELE, imam.GENOTYPE, " +
        					"ied.NAME as procedure_name, " +
        					"allele.symbol, allele.name, allele.acc, allele.gf_acc " +
        				"FROM IMA_IMAGE_RECORD iir " +
        					"LEFT OUTER JOIN IMA_IMAGE_TAG iit ON iit.IMAGE_RECORD_ID=iir.ID " +
        					"LEFT OUTER JOIN ANN_ANNOTATION aa ON aa.FOREIGN_KEY_ID=iit.ID " +
        					"LEFT OUTER JOIN IMPC_MOUSE_ALLELE_MV imam on imam.MOUSE_ID=iir.FOREIGN_KEY_ID " +
        					"LEFT OUTER JOIN IMA_SUBCONTEXT isub ON iir.subcontext_id=isub.id " +
        					"LEFT OUTER JOIN IMA_EXPERIMENT_DICT ied ON isub.experiment_dict_id=ied.id " +
        					"LEFT OUTER JOIN allele ON allele.symbol=imam.ALLELE " + 
        				"WHERE ied.NAME != 'Mouse Necropsy' " +
        				"ORDER BY iir.ID;";
        
        try {
        	
        	PreparedStatement statement = dataSource.getConnection().prepareStatement(command);
     		ResultSet res = statement.executeQuery();
                  	
			Doc doc = new Doc();			
			int i = 0;
			
	        while ( res.next()){

	        	if (res.getString("gf_acc") != null && res.getString("gf_acc").startsWith("MGI:")){
    				
		        	boolean sameImage = true;
		        	
		        	String internalId =  "komp2_" + res.getString("ID");
		        	
		        	String imageId = res.getString("ID");		        
		    		
		    		String url = "http://www.mousephenotype.org/data/media/" + res.getString("LARGE_THUMBNAIL_FILE_PATH") ;
		    		Map<String, Integer> dimensions = EnrichingUtils.getImageMeasuresFromUrl(url);
		    		
		    		if (dimensions != null){// the image could be loaded 	    		

			    		Boolean expression = false;
			    		Boolean phenotypeAnatomy = false;
		    			Boolean isMutant = true;
			    		
		    			Image image = new Image();
			    		image.setId(internalId);
			    		
			    		Dimensions d = new Dimensions();
		    			d.setImageHeight(dimensions.get("height"));
		    			d.setImageWidth(dimensions.get("width"));
			    		String procedure = res.getString("procedure_name");
			    		// store procedure name as observation
			    		image.setObservations(new StringArray());
			    		image.getObservations().getEl().add("Procedure name: " + procedure);

			    		Organism organism = new Organism();
			    		Sex sex = Sex.fromValue(norm.normalizeSex(res.getString("GENDER")));
			    		organism.setSex(sex);
			    		organism.setOrganismId(res.getString("MOUSE_NAME"));
			    		Age age = new Age();
			    		if (ageIsRelevant(procedure)){
			    			if (norm.isEmbryonicAge(res.getString("AGE_IN_WEEKS"))){
				    			age.setEmbryonicAge(norm.getAgeInDays(res.getString("AGE_IN_WEEKS")));
				    		} else {
				    			age.setAgeSinceBirth(norm.getAgeInDays(res.getString("AGE_IN_WEEKS")));
				    		}
				    		organism.setAge(age);
			    		}
			    		OntologyTerm stageOt = getStageFromProcedureOrAge(procedure, res.getString("AGE_IN_WEEKS"));
			    		if (stageOt != null){
			    			organism.setStage(stageOt);
			    		}
			    			
			    		organism.setTaxon("Mus musculus");
			    		image.setOrganism(organism);
		  	    		GenotypeComponent gt = new GenotypeComponent();
			    		
			    		if (res.getString("GENOTYPE").equalsIgnoreCase("WT")){
			    			isMutant = false;
			    		} else {
				    		gt.setGeneId(res.getString("gf_acc"));
				    		gt.setGeneSymbol(res.getString("GENE"));
				    		gt.setGeneticFeatureId(res.getString("acc"));
				    		gt.setGeneticFeatureSymbol(res.getString("ALLELE"));
				    		Zygosity zyg = Zygosity.fromValue(norm.normalizeZygosity(res.getString("GENOTYPE")));
				    		gt.setZygosity(zyg);
				    		Genotype gta = new Genotype();
				    		gta.getEl().add(gt);
				    		image.setMutantGenotypeTraits(gta);
			    		}			    		
				        
				        /* 	Channel 	*/
			    		String imageType = norm.getImageType(res.getString("procedure_name"));
			    		Channel channel = null;
			    		String channelId = "";
			    		if (imageType.equalsIgnoreCase("expression")){
			    			expression = true;
			    			channelId = internalId.replace("komp2_", "komp2_channel_") + "_" + 0; // we know that for Sanger data there is at most one channel.
			    			channel = new Channel();
					        channel.setAssociatedImage(internalId);
					        // KOMP2 always has at most 1 channel
					        StringArray c = new StringArray();
					        c.getEl().add(channelId);
					        image.setAssociatedChannel(c);
				    		channel.setId(channelId);
				    		if (isMutant){
				    			channel.setDepictsExpressionOf(gt);
				    		}
		    			}
			    			    			

				        /*	ROI	*/
			    		int k = 0;
			    		// Go through all annotations for the same image
{'fraction_non_alphanumeric': 0.1177610333692142, 'fraction_numerical': 0.0014352350197344816, 'mean_word_length': 2.5587334014300307, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3523371', 'n_tokens_mistral': 5675, 'n_tokens_neox': 4810, 'n_words': 768}