<commit_before>package controllers;

import static nl.idgis.publisher.database.QConstants.constants;
import static nl.idgis.publisher.database.QDataset.dataset;
import static nl.idgis.publisher.database.QEnvironment.environment;
import static nl.idgis.publisher.database.QPublishedService.publishedService;
import static nl.idgis.publisher.database.QPublishedServiceDataset.publishedServiceDataset;
import static nl.idgis.publisher.database.QPublishedServiceKeyword.publishedServiceKeyword;
import static nl.idgis.publisher.database.QService.service;
import static nl.idgis.publisher.database.QSourceDataset.sourceDataset;
import static nl.idgis.publisher.database.QSourceDatasetVersion.sourceDatasetVersion;

import java.sql.Timestamp;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;

import javax.inject.Inject;
import javax.xml.namespace.QName;

import com.fasterxml.jackson.databind.JsonNode;
import com.mysema.query.Tuple;
import com.mysema.query.sql.SQLQuery;
import com.mysema.query.sql.SQLSubQuery;

import nl.idgis.dav.model.DefaultResource;
import nl.idgis.dav.model.DefaultResourceDescription;
import nl.idgis.dav.model.DefaultResourceProperties;
import nl.idgis.dav.model.Resource;
import nl.idgis.dav.model.ResourceDescription;
import nl.idgis.dav.model.ResourceProperties;
import nl.idgis.publisher.metadata.MetadataDocument;
import nl.idgis.publisher.metadata.MetadataDocument.Topic;
import nl.idgis.publisher.metadata.MetadataDocumentFactory;
import nl.idgis.publisher.xml.exceptions.NotFound;
import play.libs.Json;
import util.MetadataConfig;
import util.QueryDSL;
import util.Security;
import util.QueryDSL.Transaction;

public class ServiceMetadata extends AbstractMetadata {
	
	private static final String ENDPOINT_CODE_LIST_VALUE = "WebServices";

	private static final String ENDPOINT_CODE_LIST = "http://www.isotc211.org/2005/iso19119/resources/Codelist/gmxCodelists.xml#DCPList";

	private static final String ENDPOINT_OPERATION_NAME = "GetCapabilitities";
		
	private final MetadataDocument template;
	
	@Inject
	public ServiceMetadata(MetadataConfig config, QueryDSL q, Security s) throws Exception {
		this(config, q, s, getTemplate(), "/");
	}
	
	public ServiceMetadata(MetadataConfig config, QueryDSL q, Security s, MetadataDocument template, String prefix) {
		super(config, q, s, prefix);
		
		this.template = template;
	}
	
	private static MetadataDocument getTemplate() throws Exception {
		MetadataDocumentFactory mdf = new MetadataDocumentFactory();
		
		return mdf.parseDocument(
			ServiceMetadata.class
				.getClassLoader()
				.getResourceAsStream("nl/idgis/publisher/metadata/service_metadata.xml"));
	}
	
	@Override
	public ServiceMetadata withPrefix(String prefix) {
		return new ServiceMetadata(config, q, s, template, prefix);
	}
	
	private SQLQuery fromService(Transaction tx) {
		SQLQuery query = tx.query().from(service)
			.join(publishedService).on(publishedService.serviceId.eq(service.id))
			.join(environment).on(environment.id.eq(publishedService.environmentId));
		
		if(!s.isTrusted()) {
			query.where(environment.confidential.isFalse());
		}
		
		return query;
	}

	@Override
	public Optional<Resource> resource(String name) {
		return getId(name).flatMap(id ->
			q.withTransaction(tx -> {
			
			Tuple serviceTuple = fromService(tx)
				.join(constants).on(constants.id.eq(service.constantsId))
				.where(service.wmsMetadataFileIdentification.eq(id)
					.or(service.wfsMetadataFileIdentification.eq(id)))
				.singleResult(
					service.id,
					publishedService.createTime,
					publishedService.content,
					publishedService.title,
					publishedService.alternateTitle,
					publishedService.abstractCol,
					constants.contact,
					constants.organization,
					constants.position,
					constants.addressType,
					constants.address,
					constants.city,
					constants.state,
					constants.zipcode,
					constants.country,
					constants.telephone,
					constants.fax,
					constants.email,
					service.wmsMetadataFileIdentification,
					service.wfsMetadataFileIdentification,
					environment.confidential,
					environment.url,
					environment.wmsOnly);
			
			if(serviceTuple == null) {
				return Optional.<Resource>empty();
			}
			
			JsonNode serviceInfo = Json.parse(serviceTuple.get(publishedService.content));
			boolean environmentWmsOnly = serviceTuple.get(environment.wmsOnly);
			
			if(id.equals(serviceTuple.get(service.wfsMetadataFileIdentification)) && environmentWmsOnly) {
				return Optional.<Resource>empty();
			}
			
			int serviceId = serviceTuple.get(service.id);
			
			List<String> keywords = tx.query().from(publishedServiceKeyword)
				.where(publishedServiceKeyword.serviceId.eq(serviceId))
				.orderBy(publishedServiceKeyword.keyword.asc())
				.list(publishedServiceKeyword.keyword);
			
			SQLQuery serviceDatasetQuery = tx.query().from(publishedServiceDataset)
				.join(dataset).on(dataset.id.eq(publishedServiceDataset.datasetId));
			
			if(!s.isTrusted()) {
				// do not generate links to confidential (= inaccessible) metadata documents.
				
				serviceDatasetQuery
					.join(sourceDataset).on(sourceDataset.id.eq(dataset.sourceDatasetId))
					.join(sourceDatasetVersion).on(sourceDatasetVersion.sourceDatasetId.eq(sourceDataset.id))
					.where(sourceDatasetVersion.id.in(new SQLSubQuery().from(sourceDatasetVersion)
							.where(sourceDatasetVersion.sourceDatasetId.eq(sourceDataset.id))
							.list(sourceDatasetVersion.id.max())))
					.where(sourceDatasetVersion.metadataConfidential.isFalse());
			}
			
			List<Tuple> serviceDatasetTuples = serviceDatasetQuery
				.where(publishedServiceDataset.serviceId.eq(serviceId))
				.orderBy(dataset.id.asc(), publishedServiceDataset.layerName.asc())
				.list(
					dataset.id,
					dataset.metadataIdentification,
					dataset.metadataFileIdentification,
					publishedServiceDataset.layerName);
			
			MetadataDocument metadataDocument = template.clone();
			metadataDocument.setFileIdentifier(id);
			
			metadataDocument.setServiceTitle(serviceTuple.get(publishedService.title));
			metadataDocument.setServiceAlternateTitle(serviceTuple.get(publishedService.alternateTitle));
			metadataDocument.setServiceAbstract(serviceTuple.get(publishedService.abstractCol));<commit_msg>Don't display removed datasets in service metadata<commit_after>package controllers;

import static nl.idgis.publisher.database.QConstants.constants;
import static nl.idgis.publisher.database.QDataset.dataset;
import static nl.idgis.publisher.database.QEnvironment.environment;
import static nl.idgis.publisher.database.QPublishedService.publishedService;
import static nl.idgis.publisher.database.QPublishedServiceDataset.publishedServiceDataset;
import static nl.idgis.publisher.database.QPublishedServiceKeyword.publishedServiceKeyword;
import static nl.idgis.publisher.database.QService.service;
import static nl.idgis.publisher.database.QSourceDataset.sourceDataset;
import static nl.idgis.publisher.database.QSourceDatasetVersion.sourceDatasetVersion;

import java.sql.Timestamp;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;

import javax.inject.Inject;
import javax.xml.namespace.QName;

import com.fasterxml.jackson.databind.JsonNode;
import com.mysema.query.Tuple;
import com.mysema.query.sql.SQLQuery;
import com.mysema.query.sql.SQLSubQuery;

import nl.idgis.dav.model.DefaultResource;
import nl.idgis.dav.model.DefaultResourceDescription;
import nl.idgis.dav.model.DefaultResourceProperties;
import nl.idgis.dav.model.Resource;
import nl.idgis.dav.model.ResourceDescription;
import nl.idgis.dav.model.ResourceProperties;
import nl.idgis.publisher.metadata.MetadataDocument;
import nl.idgis.publisher.metadata.MetadataDocument.Topic;
import nl.idgis.publisher.metadata.MetadataDocumentFactory;
import nl.idgis.publisher.xml.exceptions.NotFound;
import play.libs.Json;
import util.MetadataConfig;
import util.QueryDSL;
import util.Security;
import util.QueryDSL.Transaction;

public class ServiceMetadata extends AbstractMetadata {
	
	private static final String ENDPOINT_CODE_LIST_VALUE = "WebServices";

	private static final String ENDPOINT_CODE_LIST = "http://www.isotc211.org/2005/iso19119/resources/Codelist/gmxCodelists.xml#DCPList";

	private static final String ENDPOINT_OPERATION_NAME = "GetCapabilitities";
		
	private final MetadataDocument template;
	
	@Inject
	public ServiceMetadata(MetadataConfig config, QueryDSL q, Security s) throws Exception {
		this(config, q, s, getTemplate(), "/");
	}
	
	public ServiceMetadata(MetadataConfig config, QueryDSL q, Security s, MetadataDocument template, String prefix) {
		super(config, q, s, prefix);
		
		this.template = template;
	}
	
	private static MetadataDocument getTemplate() throws Exception {
		MetadataDocumentFactory mdf = new MetadataDocumentFactory();
		
		return mdf.parseDocument(
			ServiceMetadata.class
				.getClassLoader()
				.getResourceAsStream("nl/idgis/publisher/metadata/service_metadata.xml"));
	}
	
	@Override
	public ServiceMetadata withPrefix(String prefix) {
		return new ServiceMetadata(config, q, s, template, prefix);
	}
	
	private SQLQuery fromService(Transaction tx) {
		SQLQuery query = tx.query().from(service)
			.join(publishedService).on(publishedService.serviceId.eq(service.id))
			.join(environment).on(environment.id.eq(publishedService.environmentId));
		
		if(!s.isTrusted()) {
			query.where(environment.confidential.isFalse());
		}
		
		return query;
	}

	@Override
	public Optional<Resource> resource(String name) {
		return getId(name).flatMap(id ->
			q.withTransaction(tx -> {
			
			Tuple serviceTuple = fromService(tx)
				.join(constants).on(constants.id.eq(service.constantsId))
				.where(service.wmsMetadataFileIdentification.eq(id)
					.or(service.wfsMetadataFileIdentification.eq(id)))
				.singleResult(
					service.id,
					publishedService.createTime,
					publishedService.content,
					publishedService.title,
					publishedService.alternateTitle,
					publishedService.abstractCol,
					constants.contact,
					constants.organization,
					constants.position,
					constants.addressType,
					constants.address,
					constants.city,
					constants.state,
					constants.zipcode,
					constants.country,
					constants.telephone,
					constants.fax,
					constants.email,
					service.wmsMetadataFileIdentification,
					service.wfsMetadataFileIdentification,
					environment.confidential,
					environment.url,
					environment.wmsOnly);
			
			if(serviceTuple == null) {
				return Optional.<Resource>empty();
			}
			
			JsonNode serviceInfo = Json.parse(serviceTuple.get(publishedService.content));
			boolean environmentWmsOnly = serviceTuple.get(environment.wmsOnly);
			
			if(id.equals(serviceTuple.get(service.wfsMetadataFileIdentification)) && environmentWmsOnly) {
				return Optional.<Resource>empty();
			}
			
			int serviceId = serviceTuple.get(service.id);
			
			List<String> keywords = tx.query().from(publishedServiceKeyword)
				.where(publishedServiceKeyword.serviceId.eq(serviceId))
				.orderBy(publishedServiceKeyword.keyword.asc())
				.list(publishedServiceKeyword.keyword);
			
			SQLQuery serviceDatasetQuery = tx.query().from(publishedServiceDataset)
				.join(dataset).on(dataset.id.eq(publishedServiceDataset.datasetId));
			
			if(!s.isTrusted()) {
				// do not generate links to confidential (= inaccessible) metadata documents.
				
				serviceDatasetQuery
					.join(sourceDataset).on(sourceDataset.id.eq(dataset.sourceDatasetId))
					.join(sourceDatasetVersion).on(sourceDatasetVersion.sourceDatasetId.eq(sourceDataset.id))
					.where(sourceDatasetVersion.id.in(new SQLSubQuery().from(sourceDatasetVersion)
							.where(sourceDatasetVersion.sourceDatasetId.eq(sourceDataset.id))
							.list(sourceDatasetVersion.id.max())))
					.where(sourceDatasetVersion.metadataConfidential.isFalse());
			}
			
			List<Tuple> serviceDatasetTuples = serviceDatasetQuery
				.join(sourceDataset).on(sourceDataset.id.eq(dataset.sourceDatasetId))
				.where(publishedServiceDataset.serviceId.eq(serviceId)
					.and(sourceDataset.deleteTime.isNull()))
				.orderBy(dataset.id.asc(), publishedServiceDataset.layerName.asc())
				.list(
					dataset.id,
					dataset.metadataIdentification,
					dataset.metadataFileIdentification,
					publishedServiceDataset.layerName);
			
			MetadataDocument metadataDocument = template.clone();
			metadataDocument.setFileIdentifier(id);
			
			metadataDocument.setServiceTitle(serviceTuple.get(publishedService.title));
			metadataDocument.setServiceAlternateTitle(serviceTuple.get(publishedService.alternateTitle));
			metadataDocument.setServiceAbstract(serviceTuple.get(publishedService.abstractCol));
{'fraction_non_alphanumeric': 0.10257410296411856, 'fraction_numerical': 0.00187207488299532, 'mean_word_length': 5.093631178707224, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2010311', 'n_tokens_mistral': 4305, 'n_tokens_neox': 3760, 'n_words': 571}