<commit_before>package gov.nih.nci.eagle.service.strategies;

import gov.nih.nci.caintegrator.analysis.messaging.GLMSampleGroup;
import gov.nih.nci.caintegrator.analysis.messaging.GeneralizedLinearModelRequest;
import gov.nih.nci.caintegrator.analysis.messaging.SampleGroup;
import gov.nih.nci.caintegrator.application.analysis.AnalysisServerClientManager;
import gov.nih.nci.caintegrator.application.cache.BusinessTierCache;
import gov.nih.nci.caintegrator.application.service.strategy.AsynchronousFindingStrategy;
import gov.nih.nci.caintegrator.dto.query.ClassComparisonQueryDTO;
import gov.nih.nci.caintegrator.dto.query.QueryDTO;
import gov.nih.nci.caintegrator.enumeration.ArrayPlatformType;
import gov.nih.nci.caintegrator.enumeration.CoVariateType;
import gov.nih.nci.caintegrator.enumeration.StatisticalMethodType;
import gov.nih.nci.caintegrator.exceptions.FindingsAnalysisException;
import gov.nih.nci.caintegrator.exceptions.FindingsQueryException;
import gov.nih.nci.caintegrator.exceptions.ValidationException;
import gov.nih.nci.caintegrator.service.findings.Finding;
import gov.nih.nci.caintegrator.service.findings.GeneralizedLinearModelFinding;
import gov.nih.nci.caintegrator.service.task.Task;
import gov.nih.nci.caintegrator.service.task.TaskResult;
import gov.nih.nci.caintegrator.util.ValidationUtility;
import gov.nih.nci.eagle.dto.de.CoVariateDE;
import gov.nih.nci.eagle.query.dto.ClassComparisonQueryDTOImpl;
import gov.nih.nci.eagle.util.PatientGroupManager;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import javax.jms.JMSException;

import org.apache.log4j.Logger;

public class GeneralizedLinearModelFindingStrategy extends
		AsynchronousFindingStrategy {
	
	 private static Logger logger = Logger.getLogger(GeneralizedLinearModelFindingStrategy.class);
	 private Collection<SampleGroup> sampleGroups = new ArrayList<SampleGroup>();
	 private List<CoVariateType> coVariateTypes = new ArrayList<CoVariateType>();
	 private GeneralizedLinearModelRequest glmRequest = null;
	 private AnalysisServerClientManager analysisServerClientManager;
	 private Map<ArrayPlatformType, String> dataFileMap;
	 
	 private PatientGroupManager patientGroupManager;
	
	 /*
	     * (non-Javadoc)
	     * 
	     * @see gov.nih.nci.caintegrator.service.findings.strategies.FindingStrategy#createQuery()
	     *      This method validates that 2 groups were passed for generalized linear model as the statistical method
	     */
	    public boolean createQuery() throws FindingsQueryException {

	      
	        // because each layer is valid I am assured I will be getting a fulling
	        // populated query object
	        StatisticalMethodType statisticType = getQueryDTO()
	                .getStatisticTypeDE().getValueObject();

	        if (getQueryDTO().getComparisonGroups() != null) {
	        	
	        	if(statisticType==StatisticalMethodType.GLM) {	           
	                if (getQueryDTO().getComparisonGroups().size() < 1 && getQueryDTO().getBaselineGroupMap().size()>0) {
	                    throw new FindingsQueryException(
	                            "Incorrect Number of queries passed for the Generalized Linear Model statistical type");
	                }	               
	        	}
	            return true;
	        }
	        return false;
	    }
	    
	    protected void executeStrategy() {			
			
	    	glmRequest = new GeneralizedLinearModelRequest(taskResult
	                .getTask().getCacheId(), taskResult.getTask().getId());
			businessCacheManager.addToSessionCache(getTaskResult().getTask()
	                .getCacheId(), getTaskResult().getTask().getId(),
	                 getTaskResult());
		}
		
	    
	    /*
		 * 
		 */
		public boolean analyzeResultSet() throws  FindingsAnalysisException {
			StatisticalMethodType statisticType = getQueryDTO().getStatisticTypeDE().getValueObject();
			
			try {
			    if(statisticType==StatisticalMethodType.GLM) {	
			    	
			    	 // set statistical method
	                glmRequest.setStatisticalMethod(statisticType);

					 // set Co-variates (0 or more)	                
	                List<CoVariateDE> coVariateDEs = getQueryDTO().getCoVariateDEs();
	                //what?  convert from DE back to enum? why?
	                if(coVariateDEs!=null){
		                Object[] obj = coVariateDEs.toArray();
		                for(int i=0; i<obj.length;i++) {
		                	CoVariateDE coVariateDE = (CoVariateDE)obj[i];
		                	coVariateTypes.add(coVariateDE.getValueObject());	                	
		                }
	                }
	                glmRequest.setCoVariateTypes(coVariateTypes);              

			    	 // Set sample groups
	                HashMap<String, List> comparisonGroupsMap = getQueryDTO().getComparisonGroupsMap();
	                HashMap<String, List> baselineGroupMap = getQueryDTO().getBaselineGroupMap();
	                GLMSampleGroup baseline  = null;
	                for(String gname : baselineGroupMap.keySet()) {
	                	List<String> groups = baselineGroupMap.get(gname);
	                	 baseline = new GLMSampleGroup(gname);
	                	 baseline.addAll(baselineGroupMap.get(gname));
	                	 
	                	for(String name : groups )	{                   	
		                    //add each patient
		                    
		                    HashMap<String, String> annotationMap = new HashMap<String, String>();
		                    //fetch the data about each patient
		                    Map pm = patientGroupManager.getPatientInfo(name);
		                    if(coVariateTypes.contains(CoVariateType.Gender))
		                    	annotationMap.put("sex", pm.get("sex").toString());
		                    if(coVariateTypes.contains(CoVariateType.Age))
		                    	annotationMap.put("age", pm.get("age").toString());
		                    if(coVariateTypes.contains(CoVariateType.SmokingStatus))
		                    	annotationMap.put("smoking_status", pm.get("smoking_status").toString());
	
		                    baseline.addPatientData(name, annotationMap);
	                	}
	                }
                    glmRequest.setBaselineGroup(baseline);

	                
	                List<GLMSampleGroup> glmsgs = new ArrayList<GLMSampleGroup>();
	                for(String gname : comparisonGroupsMap.keySet()) {
	                	List<String> groups = comparisonGroupsMap.get(gname);
	                	GLMSampleGroup comparison = new GLMSampleGroup(gname);
	                	comparison.addAll(comparisonGroupsMap.get(gname));
	                	for(String name : groups )	{
		                    
		                    HashMap<String, String> annotationMap = new HashMap<String, String>();
		                    Map pm = patientGroupManager.getPatientInfo(name);
		                    if(coVariateTypes.contains(CoVariateType.Gender))
		                    	annotationMap.put("sex", pm.get("sex").toString());
		                    if(coVariateTypes.contains(CoVariateType.Age))
		                    	annotationMap.put("age", pm.get("age").toString());
		                    if(coVariateTypes.contains(CoVariateType.SmokingStatus))
		                    	annotationMap.put("smoking_status", pm.get("smoking_status").toString());
		                    comparison.addPatientData(name, annotationMap);
	                	}
	                	glmsgs.add(comparison);
	                }
	                
                    glmRequest.setComparisonGroups(glmsgs);
	               				
				    // set Multiple Comparison Adjustment type
	                glmRequest.setMultiGrpComparisonAdjType(getQueryDTO().getMultiGroupComparisonAdjustmentTypeDE().getValueObject());
				
				    // set foldChange
	                //glmRequest.setFoldChangeThreshold(getQueryDTO().getExprFoldChangeDE().getValueObject());
				
				    // set pvalue
	                //glmRequest.setPValueThreshold(getQueryDTO().getStatisticalSignificanceDE().getValueObject());
				
				    // set arrayplat form, come back to this to figure out how to pass the platform
	               // glmRequest.setArrayPlatform(getQueryDTO().getArrayPlatformDE().getValueObjectAsArrayPlatformType());
				
				    // go the correct matrix to fetch data			
				
	                glmRequest.setDataFileName(dataFileMap.get(getQueryDTO().getSpecimenTypeEnum().name()));
			      
			   analysisServerClientManager.sendRequest(glmRequest);
			   return true;	
			    }
			}// end of try
			catch(JMSException ex) {
				logger.error(ex.getMessage());
	  			throw new FindingsAnalysisException(ex.getMessage());
			}
			catch(Exception ex) {
				logger.error("error in glm", ex);
				throw new FindingsAnalysisException("Error in setting glmRequest object");
			}
			return false;
		}
		
<commit_msg>added filter threshold<commit_after>package gov.nih.nci.eagle.service.strategies;

import gov.nih.nci.caintegrator.analysis.messaging.GLMSampleGroup;
import gov.nih.nci.caintegrator.analysis.messaging.GeneralizedLinearModelRequest;
import gov.nih.nci.caintegrator.analysis.messaging.SampleGroup;
import gov.nih.nci.caintegrator.application.analysis.AnalysisServerClientManager;
import gov.nih.nci.caintegrator.application.cache.BusinessTierCache;
import gov.nih.nci.caintegrator.application.service.strategy.AsynchronousFindingStrategy;
import gov.nih.nci.caintegrator.dto.query.ClassComparisonQueryDTO;
import gov.nih.nci.caintegrator.dto.query.QueryDTO;
import gov.nih.nci.caintegrator.enumeration.ArrayPlatformType;
import gov.nih.nci.caintegrator.enumeration.CoVariateType;
import gov.nih.nci.caintegrator.enumeration.StatisticalMethodType;
import gov.nih.nci.caintegrator.exceptions.FindingsAnalysisException;
import gov.nih.nci.caintegrator.exceptions.FindingsQueryException;
import gov.nih.nci.caintegrator.exceptions.ValidationException;
import gov.nih.nci.caintegrator.service.findings.Finding;
import gov.nih.nci.caintegrator.service.findings.GeneralizedLinearModelFinding;
import gov.nih.nci.caintegrator.service.task.Task;
import gov.nih.nci.caintegrator.service.task.TaskResult;
import gov.nih.nci.caintegrator.util.ValidationUtility;
import gov.nih.nci.eagle.dto.de.CoVariateDE;
import gov.nih.nci.eagle.query.dto.ClassComparisonQueryDTOImpl;
import gov.nih.nci.eagle.util.PatientGroupManager;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import javax.jms.JMSException;

import org.apache.log4j.Logger;

public class GeneralizedLinearModelFindingStrategy extends
		AsynchronousFindingStrategy {
	
	 private static Logger logger = Logger.getLogger(GeneralizedLinearModelFindingStrategy.class);
	 private Collection<SampleGroup> sampleGroups = new ArrayList<SampleGroup>();
	 private List<CoVariateType> coVariateTypes = new ArrayList<CoVariateType>();
	 private GeneralizedLinearModelRequest glmRequest = null;
	 private AnalysisServerClientManager analysisServerClientManager;
	 private Map<ArrayPlatformType, String> dataFileMap;
     
     private static final Double variance = .95;
	 
	 private PatientGroupManager patientGroupManager;
	
	 /*
	     * (non-Javadoc)
	     * 
	     * @see gov.nih.nci.caintegrator.service.findings.strategies.FindingStrategy#createQuery()
	     *      This method validates that 2 groups were passed for generalized linear model as the statistical method
	     */
	    public boolean createQuery() throws FindingsQueryException {

	      
	        // because each layer is valid I am assured I will be getting a fulling
	        // populated query object
	        StatisticalMethodType statisticType = getQueryDTO()
	                .getStatisticTypeDE().getValueObject();

	        if (getQueryDTO().getComparisonGroups() != null) {
	        	
	        	if(statisticType==StatisticalMethodType.GLM) {	           
	                if (getQueryDTO().getComparisonGroups().size() < 1 && getQueryDTO().getBaselineGroupMap().size()>0) {
	                    throw new FindingsQueryException(
	                            "Incorrect Number of queries passed for the Generalized Linear Model statistical type");
	                }	               
	        	}
	            return true;
	        }
	        return false;
	    }
	    
	    protected void executeStrategy() {			
			
	    	glmRequest = new GeneralizedLinearModelRequest(taskResult
	                .getTask().getCacheId(), taskResult.getTask().getId());
			businessCacheManager.addToSessionCache(getTaskResult().getTask()
	                .getCacheId(), getTaskResult().getTask().getId(),
	                 getTaskResult());
		}
		
	    
	    /*
		 * 
		 */
		public boolean analyzeResultSet() throws  FindingsAnalysisException {
			StatisticalMethodType statisticType = getQueryDTO().getStatisticTypeDE().getValueObject();
			
			try {
			    if(statisticType==StatisticalMethodType.GLM) {	
			    	
			    	 // set statistical method
	                glmRequest.setStatisticalMethod(statisticType);

					 // set Co-variates (0 or more)	                
	                List<CoVariateDE> coVariateDEs = getQueryDTO().getCoVariateDEs();
	                //what?  convert from DE back to enum? why?
	                if(coVariateDEs!=null){
		                Object[] obj = coVariateDEs.toArray();
		                for(int i=0; i<obj.length;i++) {
		                	CoVariateDE coVariateDE = (CoVariateDE)obj[i];
		                	coVariateTypes.add(coVariateDE.getValueObject());	                	
		                }
	                }
	                glmRequest.setCoVariateTypes(coVariateTypes);              

			    	 // Set sample groups
	                HashMap<String, List> comparisonGroupsMap = getQueryDTO().getComparisonGroupsMap();
	                HashMap<String, List> baselineGroupMap = getQueryDTO().getBaselineGroupMap();
	                GLMSampleGroup baseline  = null;
	                for(String gname : baselineGroupMap.keySet()) {
	                	List<String> groups = baselineGroupMap.get(gname);
	                	 baseline = new GLMSampleGroup(gname);
	                	 baseline.addAll(baselineGroupMap.get(gname));
	                	 
	                	for(String name : groups )	{                   	
		                    //add each patient
		                    
		                    HashMap<String, String> annotationMap = new HashMap<String, String>();
		                    //fetch the data about each patient
		                    Map pm = patientGroupManager.getPatientInfo(name);
		                    if(coVariateTypes.contains(CoVariateType.Gender))
		                    	annotationMap.put("sex", pm.get("sex").toString());
		                    if(coVariateTypes.contains(CoVariateType.Age))
		                    	annotationMap.put("age", pm.get("age").toString());
		                    if(coVariateTypes.contains(CoVariateType.SmokingStatus))
		                    	annotationMap.put("smoking_status", pm.get("smoking_status").toString());
	
		                    baseline.addPatientData(name, annotationMap);
	                	}
	                }
                    glmRequest.setBaselineGroup(baseline);

	                
	                List<GLMSampleGroup> glmsgs = new ArrayList<GLMSampleGroup>();
	                for(String gname : comparisonGroupsMap.keySet()) {
	                	List<String> groups = comparisonGroupsMap.get(gname);
	                	GLMSampleGroup comparison = new GLMSampleGroup(gname);
	                	comparison.addAll(comparisonGroupsMap.get(gname));
	                	for(String name : groups )	{
		                    
		                    HashMap<String, String> annotationMap = new HashMap<String, String>();
		                    Map pm = patientGroupManager.getPatientInfo(name);
		                    if(coVariateTypes.contains(CoVariateType.Gender))
		                    	annotationMap.put("sex", pm.get("sex").toString());
		                    if(coVariateTypes.contains(CoVariateType.Age))
		                    	annotationMap.put("age", pm.get("age").toString());
		                    if(coVariateTypes.contains(CoVariateType.SmokingStatus))
		                    	annotationMap.put("smoking_status", pm.get("smoking_status").toString());
		                    comparison.addPatientData(name, annotationMap);
	                	}
	                	glmsgs.add(comparison);
	                }
	                
                    glmRequest.setComparisonGroups(glmsgs);
	               				
				    // set Multiple Comparison Adjustment type
	                glmRequest.setMultiGrpComparisonAdjType(getQueryDTO().getMultiGroupComparisonAdjustmentTypeDE().getValueObject());
				
	                // set gene variance 
                    glmRequest.setGeneVariance(variance);
				
				    // go the correct matrix to fetch data			
				
	                glmRequest.setDataFileName(dataFileMap.get(getQueryDTO().getSpecimenTypeEnum().name()));
			      
			   analysisServerClientManager.sendRequest(glmRequest);
			   return true;	
			    }
			}// end of try
			catch(JMSException ex) {
				logger.error(ex.getMessage());
	  			throw new FindingsAnalysisException(ex.getMessage());
			}
			catch(Exception ex) {
				logger.error("error in glm", ex);
				throw new FindingsAnalysisException("Error in setting glmRequest object");
			}
			return false;
		}
		

{'fraction_non_alphanumeric': 0.09346756663997255, 'fraction_numerical': 0.0008008237043816497, 'mean_word_length': 2.3218696560896825, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '537576', 'n_tokens_mistral': 5270, 'n_tokens_neox': 4586, 'n_words': 754}