<commit_before>package dk.statsbiblioteket.doms.licensemodule.validation;

import java.util.*;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import dk.statsbiblioteket.doms.licensemodule.LicenseModulePropertiesLoader;
import dk.statsbiblioteket.doms.licensemodule.Util;
import dk.statsbiblioteket.doms.licensemodule.persistence.Attribute;
import dk.statsbiblioteket.doms.licensemodule.persistence.AttributeGroup;
import dk.statsbiblioteket.doms.licensemodule.persistence.AttributeValue;
import dk.statsbiblioteket.doms.licensemodule.persistence.ConfiguredDomLicenseGroupType;
import dk.statsbiblioteket.doms.licensemodule.persistence.ConfiguredDomLicensePresentationType;
import dk.statsbiblioteket.doms.licensemodule.persistence.License;
import dk.statsbiblioteket.doms.licensemodule.persistence.LicenseCache;
import dk.statsbiblioteket.doms.licensemodule.persistence.LicenseContent;
import dk.statsbiblioteket.doms.licensemodule.persistence.Presentation;
import dk.statsbiblioteket.doms.licensemodule.service.dto.CheckAccessForIdsInputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.CheckAccessForIdsOutputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.GetUserGroupsInputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.GetUserQueryInputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.GetUserQueryOutputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.GetUsersLicensesInputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.UserGroupDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.UserObjAttributeDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.ValidateAccessInputDTO;
import dk.statsbiblioteket.doms.licensemodule.solr.SolrServerClient;

public class LicenseValidator {

	private static final Logger log = LoggerFactory.getLogger(LicenseValidator.class);		
	public static final String LOCALE_DA = "da";
	public static final String LOCALE_EN = "en";
	public static final String NO_ACCESS = "(recordID:NoAccess)";

	
	
    public static final List locales=
			     Collections.unmodifiableList(Arrays.asList(new String[] {LOCALE_DA,LOCALE_EN}));
	
	// The following 3 methods are the API

	//TODO shitload of javadoc
	public static ArrayList<License> getUsersLicenses(GetUsersLicensesInputDTO input) throws Exception{
		//validate
		if (input.getAttributes() == null || input.getAttributes().size() == 0){
			log.error("No attributes defined in input.");
			throw new IllegalArgumentException("No attributes defined in input");
		}
        validateLocale(input.getLocale());

		// First filter by valid date
		ArrayList<License> allLicenses = LicenseCache.getAllLicense();		
		ArrayList<License> dateFilteredLicenses = filterLicenseByValidDate(allLicenses, System.currentTimeMillis());		

		//Find licenses that give access (not checking groups) for the dateFiltered licenses
		ArrayList<License> accessLicenses = findLicensesValidatingAccess(input.getAttributes(),  dateFilteredLicenses);

		return accessLicenses;
	}

	//TODO shitload of javadoc
	public static ArrayList<UserGroupDTO> getUsersGroups(GetUserGroupsInputDTO input) throws Exception{
		//validate
		if (input.getAttributes() == null || input.getAttributes().size() == 0){
			log.error("No attributes defined in input.");
			throw new IllegalArgumentException("No attributes defined in input");
		}

		validateLocale(input.getLocale());
		
		// First filter by valid date
		ArrayList<License> allLicenses = LicenseCache.getAllLicense();		

		ArrayList<License> dateFilteredLicenses = filterLicenseByValidDate(allLicenses, System.currentTimeMillis());		
		
		//Find licenses that give access (not checking groups) for the dateFiltered licenses
		ArrayList<License> accessLicenses = findLicensesValidatingAccess(input.getAttributes(),  dateFilteredLicenses);

		ArrayList<UserGroupDTO> filteredGroups = filterGroupsWithPresentationtype(accessLicenses);
		        
	    LicenseValidator.fixLocale(filteredGroups, input.getLocale());       
	    
		return filteredGroups;
	}

	//TODO shitload of javadoc
	public static CheckAccessForIdsOutputDTO checkAccessForIds(CheckAccessForIdsInputDTO input) throws Exception{
       
		//Get the query. This also validates the input 
		GetUserQueryInputDTO inputQuery = new GetUserQueryInputDTO();		
		inputQuery.setAttributes(input.getAttributes());
		inputQuery.setPresentationType(input.getPresentationType());
		GetUserQueryOutputDTO query = getUserQuery(inputQuery);
		
		
		CheckAccessForIdsOutputDTO output = new  CheckAccessForIdsOutputDTO();
		output.setPresentationType(input.getPresentationType());
		output.setQuery(query.getQuery());

		ArrayList<SolrServerClient> servers = LicenseModulePropertiesLoader.SOLR_SERVERS;
	
		  // merge (union) results.   
        Set<String> filteredIdsSet = new HashSet<String>();
		
		//Next step: use Future's to make multithreaded when we get more servers. 
		//But currently these requests are less 10 ms
		for (SolrServerClient server: servers){
	        ArrayList<String> filteredIds =server.filterIds(input.getIds(), query.getQuery());
	        log.debug("#filtered id for server "+server.getSolrServer().getBaseURL() +" : "+filteredIds.size());
		    filteredIdsSet.addAll(filteredIds);
		}
		//Now we have to remove remove ID's not asked for that are here because of multivalue field. (set intersection)
	    filteredIdsSet.retainAll(input.getIds());
						
		output.setAccessIds(new ArrayList<String>(filteredIdsSet));
		//Sanity check!
		if (output.getAccessIds().size() > input.getIds().size()){
			throw new IllegalArgumentException("Security problem: More Id's in output than input. Check for query injection.");
		}
		
		log.debug("#query ID:"+input.getIds().size() + " #filtered IDs:"+filteredIdsSet.size());
		return output;		
	}

	//TODO shitload of javadoc
	public static GetUserQueryOutputDTO getUserQuery(GetUserQueryInputDTO input) throws Exception{
		//validate
		if (input.getAttributes() == null){
			log.error("No attributes defined in input.");
			input.setAttributes(new ArrayList<UserObjAttributeDTO>());
			
		}		

		if (input.getPresentationType() == null){
			log.error("No presentationtype defined in input.");
			throw new IllegalArgumentException("No presentationtype defined in input.");
		}


		//Will throw exception if not matched		
        matchPresentationtype(input.getPresentationType());		


		// First filter by valid date
		ArrayList<License> allLicenses = LicenseCache.getAllLicense();		
		ArrayList<License> dateFilteredLicenses = filterLicenseByValidDate(allLicenses, System.currentTimeMillis());		

		//Find licenses that give access (not checking groups) for the dateFiltered licenses
		ArrayList<License> accessLicenses = findLicensesValidatingAccess(input.getAttributes(),  dateFilteredLicenses);

		ArrayList<String> types = new ArrayList<String>();
		types.add(input.getPresentationType());<commit_msg>small debug change<commit_after>package dk.statsbiblioteket.doms.licensemodule.validation;

import java.util.*;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import dk.statsbiblioteket.doms.licensemodule.LicenseModulePropertiesLoader;
import dk.statsbiblioteket.doms.licensemodule.Util;
import dk.statsbiblioteket.doms.licensemodule.persistence.Attribute;
import dk.statsbiblioteket.doms.licensemodule.persistence.AttributeGroup;
import dk.statsbiblioteket.doms.licensemodule.persistence.AttributeValue;
import dk.statsbiblioteket.doms.licensemodule.persistence.ConfiguredDomLicenseGroupType;
import dk.statsbiblioteket.doms.licensemodule.persistence.ConfiguredDomLicensePresentationType;
import dk.statsbiblioteket.doms.licensemodule.persistence.License;
import dk.statsbiblioteket.doms.licensemodule.persistence.LicenseCache;
import dk.statsbiblioteket.doms.licensemodule.persistence.LicenseContent;
import dk.statsbiblioteket.doms.licensemodule.persistence.Presentation;
import dk.statsbiblioteket.doms.licensemodule.service.dto.CheckAccessForIdsInputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.CheckAccessForIdsOutputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.GetUserGroupsInputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.GetUserQueryInputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.GetUserQueryOutputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.GetUsersLicensesInputDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.UserGroupDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.UserObjAttributeDTO;
import dk.statsbiblioteket.doms.licensemodule.service.dto.ValidateAccessInputDTO;
import dk.statsbiblioteket.doms.licensemodule.solr.SolrServerClient;

public class LicenseValidator {

	private static final Logger log = LoggerFactory.getLogger(LicenseValidator.class);		
	public static final String LOCALE_DA = "da";
	public static final String LOCALE_EN = "en";
	public static final String NO_ACCESS = "(recordID:NoAccess)";

	
	
    public static final List locales=
			     Collections.unmodifiableList(Arrays.asList(new String[] {LOCALE_DA,LOCALE_EN}));
	
	// The following 3 methods are the API

	//TODO shitload of javadoc
	public static ArrayList<License> getUsersLicenses(GetUsersLicensesInputDTO input) throws Exception{
		//validate
		if (input.getAttributes() == null || input.getAttributes().size() == 0){
			log.error("No attributes defined in input.");
			throw new IllegalArgumentException("No attributes defined in input");
		}
        validateLocale(input.getLocale());

		// First filter by valid date
		ArrayList<License> allLicenses = LicenseCache.getAllLicense();		
		ArrayList<License> dateFilteredLicenses = filterLicenseByValidDate(allLicenses, System.currentTimeMillis());		

		//Find licenses that give access (not checking groups) for the dateFiltered licenses
		ArrayList<License> accessLicenses = findLicensesValidatingAccess(input.getAttributes(),  dateFilteredLicenses);

		return accessLicenses;
	}

	//TODO shitload of javadoc
	public static ArrayList<UserGroupDTO> getUsersGroups(GetUserGroupsInputDTO input) throws Exception{
		//validate
		if (input.getAttributes() == null || input.getAttributes().size() == 0){
			log.error("No attributes defined in input.");
			throw new IllegalArgumentException("No attributes defined in input");
		}

		validateLocale(input.getLocale());
		
		// First filter by valid date
		ArrayList<License> allLicenses = LicenseCache.getAllLicense();		

		ArrayList<License> dateFilteredLicenses = filterLicenseByValidDate(allLicenses, System.currentTimeMillis());		
		
		//Find licenses that give access (not checking groups) for the dateFiltered licenses
		ArrayList<License> accessLicenses = findLicensesValidatingAccess(input.getAttributes(),  dateFilteredLicenses);

		ArrayList<UserGroupDTO> filteredGroups = filterGroupsWithPresentationtype(accessLicenses);
		        
	    LicenseValidator.fixLocale(filteredGroups, input.getLocale());       
	    
		return filteredGroups;
	}

	//TODO shitload of javadoc
	public static CheckAccessForIdsOutputDTO checkAccessForIds(CheckAccessForIdsInputDTO input) throws Exception{
       
		//Get the query. This also validates the input 
		GetUserQueryInputDTO inputQuery = new GetUserQueryInputDTO();		
		inputQuery.setAttributes(input.getAttributes());
		inputQuery.setPresentationType(input.getPresentationType());
		GetUserQueryOutputDTO query = getUserQuery(inputQuery);
		
		
		CheckAccessForIdsOutputDTO output = new  CheckAccessForIdsOutputDTO();
		output.setPresentationType(input.getPresentationType());
		output.setQuery(query.getQuery());

		ArrayList<SolrServerClient> servers = LicenseModulePropertiesLoader.SOLR_SERVERS;
	
		  // merge (union) results.   
        Set<String> filteredIdsSet = new HashSet<String>();
		
		//Next step: use Future's to make multithreaded when we get more servers. 
		//But currently these requests are less 10 ms
		for (SolrServerClient server: servers){
	        ArrayList<String> filteredIds =server.filterIds(input.getIds(), query.getQuery());
	        log.debug("#filtered id for server "+server.getSolrServer().getBaseURL() +" : "+filteredIds.size());
		    filteredIdsSet.addAll(filteredIds);
		}
		//Now we have to remove remove ID's not asked for that are here because of multivalue field. (set intersection)
	    filteredIdsSet.retainAll(input.getIds());
						
		output.setAccessIds(new ArrayList<String>(filteredIdsSet));
		//Sanity check!
		if (output.getAccessIds().size() > input.getIds().size()){
			throw new IllegalArgumentException("Security problem: More Id's in output than input. Check for query injection.");
		}
		
		log.debug("#query IDs="+input.getIds().size() + " returned #filtered IDs="+output.getAccessIds().size());
		return output;		
	}

	//TODO shitload of javadoc
	public static GetUserQueryOutputDTO getUserQuery(GetUserQueryInputDTO input) throws Exception{
		//validate
		if (input.getAttributes() == null){
			log.error("No attributes defined in input.");
			input.setAttributes(new ArrayList<UserObjAttributeDTO>());
			
		}		

		if (input.getPresentationType() == null){
			log.error("No presentationtype defined in input.");
			throw new IllegalArgumentException("No presentationtype defined in input.");
		}


		//Will throw exception if not matched		
        matchPresentationtype(input.getPresentationType());		


		// First filter by valid date
		ArrayList<License> allLicenses = LicenseCache.getAllLicense();		
		ArrayList<License> dateFilteredLicenses = filterLicenseByValidDate(allLicenses, System.currentTimeMillis());		

		//Find licenses that give access (not checking groups) for the dateFiltered licenses
		ArrayList<License> accessLicenses = findLicensesValidatingAccess(input.getAttributes(),  dateFilteredLicenses);

		ArrayList<String> types = new ArrayList<String>();
		types.add(input.getPresentationType());
{'fraction_non_alphanumeric': 0.09639330184628596, 'fraction_numerical': 0.0010018605982539, 'mean_word_length': 5.588873173031589, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4336739', 'n_tokens_mistral': 4336, 'n_tokens_neox': 3961, 'n_words': 846}