<commit_before>/* $This file is distributed under the terms of the license in /doc/license.txt$ */

package edu.cornell.mannlib.vitro.webapp.utils.pageDataGetter;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.servlet.ServletContext;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.json.JSONObject;

import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListController;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VClassGroupCache;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.VClassGroupTemplateModel;

/**
 * This will pass these variables to the template:
 * classGroupUri: uri of the classgroup associated with this page.
 * vClassGroup: a data structure that is the classgroup associated with this page.     
 */
public class IndividualsForClassesDataGetter implements PageDataGetter{
    private static final Log log = LogFactory.getLog(IndividualsForClassesDataGetter.class);
    protected static String restrictClassesTemplateName = "restricted";
    public Map<String,Object> getData(ServletContext context, VitroRequest vreq, String pageUri, Map<String, Object> page ){
        HashMap<String, Object> data = new HashMap<String,Object>();
        //This is the old technique of getting class intersections
        Map<String, List<String>> classIntersectionsMap = vreq.getWebappDaoFactory().getPageDao().getClassesAndRestrictionsForPage(pageUri);
        
        try{
        	List<String> classes = classIntersectionsMap.get("classes");
        	List<String> restrictClasses = classIntersectionsMap.get("restrictClasses");
        	log.debug("Retrieving classes for " + classes.toString() + " and restricting by " + restrictClasses.toString());
        	processClassesAndRestrictions(vreq, context, data, classes, restrictClasses);
        	 //Also add data service url
            //Hardcoding for now, need a more dynamic way of doing this
            data.put("dataServiceUrlIndividualsByVClass", this.getDataServiceUrl());
        } catch(Exception ex) {
        	log.error("An error occurred retrieving Vclass Intersection individuals", ex);
        }
             
        return data;
    }        
    
    protected void processClassesAndRestrictions(VitroRequest vreq, ServletContext context, 
    		HashMap<String, Object> data, List<String> classes, List<String> restrictClasses ) {
    	processClassesForDisplay(context, data, classes);
    	processRestrictionClasses(vreq, context, data, restrictClasses);
    	processIntersections(vreq, context, data);
    }
    
    //At this point, data specifices whether or not intersections included
    private void processIntersections(VitroRequest vreq,
			ServletContext context, HashMap<String, Object> data) {
    	VClassGroup classesGroup = (VClassGroup) data.get("vClassGroup");
    	List<VClass> vclassList = classesGroup.getVitroClassList();
    	List<VClass> restrictClasses = (List<VClass>) data.get("restrictVClasses");
    	//if there are restrict classes, then update counts
    	if(restrictClasses.size() > 0) {
    		List<VClass> newVClassList = new ArrayList<VClass>();
    		//Iterate through vclasses and get updated counts
    		for(VClass v: vclassList) {
    			int oldCount = v.getEntityCount();
    			//Making a copy so as to ensure we don't touch the values in the cache
    			VClass copyVClass = makeCopyVClass(v);
    			int count = retrieveCount(vreq, context, v, restrictClasses);
    			if(oldCount != count) {
    				log.debug("Old count was " + v.getEntityCount() + " and New count for " + v.getURI() + " is " + count);
    				copyVClass.setEntityCount(count);
    			} 
    			newVClassList.add(copyVClass);
    		}
    		classesGroup.setVitroClassList(newVClassList);
    		//TODO: Do we need to do this again or will this already be reset?
    		data.put("vClassGroup", classesGroup);
    	}
	}

    private VClass makeCopyVClass(VClass v) {
    	VClass copyVClass = new VClass(v.getURI());
		copyVClass.setLocalName(copyVClass.getLocalName());
		copyVClass.setDisplayRank(v.getDisplayRank());
		copyVClass.setName(v.getName());
		copyVClass.setNamespace(v.getNamespace());
		copyVClass.setEntityCount(v.getEntityCount());
		return copyVClass;
    }
    
    //update class count based on restrict classes
	private int retrieveCount(VitroRequest vreq, ServletContext context, VClass v, List<VClass> restrictClasses) {
		//Execute solr query that returns only count of individuals
		log.debug("Entity count is " + v.getEntityCount());
		List<String> classUris = new ArrayList<String>();
		classUris.add(v.getURI());
		for(VClass r: restrictClasses) {
			classUris.add(r.getURI());
		}
		long count =  DataGetterUtils.getIndividualCountForIntersection(vreq, context, classUris);
		return new Long(count).intValue();

	}

	private void processClassesForDisplay(ServletContext context, HashMap<String, Object> data, List<String> classes) {
    	VClassGroup classesGroup = new VClassGroup();
    	classesGroup.setURI("displayClasses");
    	
    	List<VClass> vClasses = new ArrayList<VClass>();
  
    	VClassGroupCache vcgc = VClassGroupCache.getVClassGroupCache(context);
    	for(String classUri: classes) {
    		//VClass vclass = vreq.getWebappDaoFactory().getVClassDao().getVClassByURI(classUri);
    		//Retrieve vclass from cache to get the count
    		VClass vclass = vcgc.getCachedVClass(classUri);
    		if(vclass != null) {
    			log.debug("VClass does exist for " + classUri + " and entity count is " + vclass.getEntityCount());
    			vClasses.add(vclass);
    		} else {
    			log.debug("Vclass " + classUri + " does not exist in the cache");
    			log.error("Error occurred, vclass does not exist for this uri " + classUri);
    			//Throw exception here
    		}
    	}
    	classesGroup.setVitroClassList(vClasses);
    	//Set vclass group
    	data.put("vClassGroup", classesGroup);
    }	
    
    private void processRestrictionClasses(VitroRequest vreq, ServletContext context, 
    		HashMap<String, Object> data, List<String> restrictClasses) {
    	try {
	    	VClassGroup restrictClassesGroup = new VClassGroup();
	    	restrictClassesGroup.setURI("restrictClasses");
	    	
	    	List<VClass> restrictVClasses = new ArrayList<VClass>();
	    	
	    	List<String> urlEncodedRestrictClasses = new ArrayList<String>();
	    	if(restrictClasses.size() > 0) {
	    		//classes for restriction are not displayed so don't need to include their class individual counts
	    		for(String restrictClassUri: restrictClasses) {
	    			VClass vclass = vreq.getWebappDaoFactory().getVClassDao().getVClassByURI(restrictClassUri);
	        		if(vclass != null) {
	        			log.debug("Found restrict class and adding to list " + restrictClassUri);
	        			restrictVClasses.add(vclass);
	        		} else {
	        			log.error("Error occurred, vclass does not exist for this uri " + restrictClassUri);
	        		}<commit_msg>Updating individuals for classes data getter to sort class list for display <commit_after>/* $This file is distributed under the terms of the license in /doc/license.txt$ */

package edu.cornell.mannlib.vitro.webapp.utils.pageDataGetter;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.servlet.ServletContext;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.json.JSONObject;

import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListController;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VClassGroupCache;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.VClassGroupTemplateModel;

/**
 * This will pass these variables to the template:
 * classGroupUri: uri of the classgroup associated with this page.
 * vClassGroup: a data structure that is the classgroup associated with this page.     
 */
public class IndividualsForClassesDataGetter implements PageDataGetter{
    private static final Log log = LogFactory.getLog(IndividualsForClassesDataGetter.class);
    protected static String restrictClassesTemplateName = "restricted";
    public Map<String,Object> getData(ServletContext context, VitroRequest vreq, String pageUri, Map<String, Object> page ){
        HashMap<String, Object> data = new HashMap<String,Object>();
        //This is the old technique of getting class intersections
        Map<String, List<String>> classIntersectionsMap = vreq.getWebappDaoFactory().getPageDao().getClassesAndRestrictionsForPage(pageUri);
        
        try{
        	List<String> classes = classIntersectionsMap.get("classes");
        	List<String> restrictClasses = classIntersectionsMap.get("restrictClasses");
        	log.debug("Retrieving classes for " + classes.toString() + " and restricting by " + restrictClasses.toString());
        	processClassesAndRestrictions(vreq, context, data, classes, restrictClasses);
        	 //Also add data service url
            //Hardcoding for now, need a more dynamic way of doing this
            data.put("dataServiceUrlIndividualsByVClass", this.getDataServiceUrl());
        } catch(Exception ex) {
        	log.error("An error occurred retrieving Vclass Intersection individuals", ex);
        }
             
        return data;
    }        
    
    protected void processClassesAndRestrictions(VitroRequest vreq, ServletContext context, 
    		HashMap<String, Object> data, List<String> classes, List<String> restrictClasses ) {
    	processClassesForDisplay(context, data, classes);
    	processRestrictionClasses(vreq, context, data, restrictClasses);
    	processIntersections(vreq, context, data);
    }
    
    //At this point, data specifices whether or not intersections included
    private void processIntersections(VitroRequest vreq,
			ServletContext context, HashMap<String, Object> data) {
    	VClassGroup classesGroup = (VClassGroup) data.get("vClassGroup");
    	List<VClass> vclassList = classesGroup.getVitroClassList();
    	List<VClass> restrictClasses = (List<VClass>) data.get("restrictVClasses");
    	//if there are restrict classes, then update counts
    	if(restrictClasses.size() > 0) {
    		List<VClass> newVClassList = new ArrayList<VClass>();
    		//Iterate through vclasses and get updated counts
    		for(VClass v: vclassList) {
    			int oldCount = v.getEntityCount();
    			//Making a copy so as to ensure we don't touch the values in the cache
    			VClass copyVClass = makeCopyVClass(v);
    			int count = retrieveCount(vreq, context, v, restrictClasses);
    			if(oldCount != count) {
    				log.debug("Old count was " + v.getEntityCount() + " and New count for " + v.getURI() + " is " + count);
    				copyVClass.setEntityCount(count);
    			} 
    			newVClassList.add(copyVClass);
    		}
    		classesGroup.setVitroClassList(newVClassList);
    		//TODO: Do we need to do this again or will this already be reset?
    		data.put("vClassGroup", classesGroup);
    	}
	}

    private VClass makeCopyVClass(VClass v) {
    	VClass copyVClass = new VClass(v.getURI());
		copyVClass.setLocalName(copyVClass.getLocalName());
		copyVClass.setDisplayRank(v.getDisplayRank());
		copyVClass.setName(v.getName());
		copyVClass.setNamespace(v.getNamespace());
		copyVClass.setEntityCount(v.getEntityCount());
		return copyVClass;
    }
    
    //update class count based on restrict classes
	private int retrieveCount(VitroRequest vreq, ServletContext context, VClass v, List<VClass> restrictClasses) {
		//Execute solr query that returns only count of individuals
		log.debug("Entity count is " + v.getEntityCount());
		List<String> classUris = new ArrayList<String>();
		classUris.add(v.getURI());
		for(VClass r: restrictClasses) {
			classUris.add(r.getURI());
		}
		long count =  DataGetterUtils.getIndividualCountForIntersection(vreq, context, classUris);
		return new Long(count).intValue();

	}

	private void processClassesForDisplay(ServletContext context, HashMap<String, Object> data, List<String> classes) {
    	VClassGroup classesGroup = new VClassGroup();
    	classesGroup.setURI("displayClasses");
    	
    	List<VClass> vClasses = new ArrayList<VClass>();
  
    	VClassGroupCache vcgc = VClassGroupCache.getVClassGroupCache(context);
    	for(String classUri: classes) {
    		//VClass vclass = vreq.getWebappDaoFactory().getVClassDao().getVClassByURI(classUri);
    		//Retrieve vclass from cache to get the count
    		VClass vclass = vcgc.getCachedVClass(classUri);
    		if(vclass != null) {
    			log.debug("VClass does exist for " + classUri + " and entity count is " + vclass.getEntityCount());
    			vClasses.add(vclass);
    		} else {
    			log.debug("Vclass " + classUri + " does not exist in the cache");
    			log.error("Error occurred, vclass does not exist for this uri " + classUri);
    			//Throw exception here
    		}
    	}
    	//Sort these classes
    	Collections.sort(vClasses);
    	classesGroup.setVitroClassList(vClasses);
    	//Set vclass group
    	data.put("vClassGroup", classesGroup);
    }	
    
    private void processRestrictionClasses(VitroRequest vreq, ServletContext context, 
    		HashMap<String, Object> data, List<String> restrictClasses) {
    	try {
	    	VClassGroup restrictClassesGroup = new VClassGroup();
	    	restrictClassesGroup.setURI("restrictClasses");
	    	
	    	List<VClass> restrictVClasses = new ArrayList<VClass>();
	    	
	    	List<String> urlEncodedRestrictClasses = new ArrayList<String>();
	    	if(restrictClasses.size() > 0) {
	    		//classes for restriction are not displayed so don't need to include their class individual counts
	    		for(String restrictClassUri: restrictClasses) {
	    			VClass vclass = vreq.getWebappDaoFactory().getVClassDao().getVClassByURI(restrictClassUri);
	        		if(vclass != null) {
	        			log.debug("Found restrict class and adding to list " + restrictClassUri);
	        			restrictVClasses.add(vclass);
	        		} else {
	        			log.error("Error occurred, vclass does not exist for this uri " + restrictClassUri);
	        		}
{'fraction_non_alphanumeric': 0.09938300272009554, 'fraction_numerical': 0.000265375174152458, 'mean_word_length': 3.779327837666455, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5568719', 'n_tokens_mistral': 4689, 'n_tokens_neox': 4506, 'n_words': 1068}