<commit_before>package it.unibz.inf.ontop.owlrefplatform.core;


import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
import it.unibz.inf.ontop.injection.NativeQueryLanguageComponentFactory;
import it.unibz.inf.ontop.model.*;
import it.unibz.inf.ontop.ontology.*;
import it.unibz.inf.ontop.owlrefplatform.core.basicoperations.CQContainmentCheckUnderLIDs;
import it.unibz.inf.ontop.owlrefplatform.core.basicoperations.EQNormalizer;
import it.unibz.inf.ontop.owlrefplatform.core.basicoperations.LinearInclusionDependencies;
import it.unibz.inf.ontop.owlrefplatform.core.basicoperations.VocabularyValidator;
import it.unibz.inf.ontop.owlrefplatform.core.dagjgrapht.TBoxReasoner;
import it.unibz.inf.ontop.owlrefplatform.core.mappingprocessing.*;
import it.unibz.inf.ontop.owlrefplatform.core.unfolding.DatalogUnfolder;

import it.unibz.inf.ontop.model.impl.OBDAVocabulary;
import it.unibz.inf.ontop.model.impl.TermUtils;
import it.unibz.inf.ontop.injection.QuestCoreSettings;
import it.unibz.inf.ontop.pivotalrepr.MetadataForQueryOptimization;
import it.unibz.inf.ontop.pivotalrepr.impl.MetadataForQueryOptimizationImpl;
import it.unibz.inf.ontop.sql.RDBMetadata;
import it.unibz.inf.ontop.sql.DatabaseRelationDefinition;
import it.unibz.inf.ontop.sql.Relation2DatalogPredicate;
import it.unibz.inf.ontop.sql.RelationID;
import it.unibz.inf.ontop.utils.IMapping2DatalogConverter;
import it.unibz.inf.ontop.utils.ImmutableCollectors;
import net.sf.jsqlparser.JSQLParserException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.SQLException;
import java.util.*;
import java.util.stream.IntStream;

import static it.unibz.inf.ontop.model.impl.OntopModelSingletons.DATA_FACTORY;

public class QuestUnfolder {

	private final NativeQueryLanguageComponentFactory nativeQLFactory;
	private final QuestCoreSettings preferences;
	/* The active unfolding engine */
	private DatalogUnfolder unfolder;

	/*
	 * These are pattern matchers that will help transforming the URI's in
	 * queries into Functions, used by the SPARQL translator.
	 */
	private UriTemplateMatcher uriTemplateMatcher = new UriTemplateMatcher();

	protected List<CQIE> ufp; // for TESTS ONLY

	private static final Logger log = LoggerFactory.getLogger(QuestUnfolder.class);

	private ImmutableMultimap<AtomPredicate, ImmutableList<Integer>> uniqueConstraints;
	private final IMapping2DatalogConverter mapping2DatalogConvertor;
	private MetadataForQueryOptimization metadataForQueryOptimization;

	private Set<Predicate> dataPropertiesAndClassesMapped = new HashSet<>();
	private Set<Predicate> objectPropertiesMapped = new HashSet<>();

	/** Davide> Whether to exclude the user-supplied predicates from the
	 *          TMapping procedure (that is, the mapping assertions for
	 *          those predicates should not be extended according to the
	 *          TBox hierarchies
	 */
	//private boolean applyExcludeFromTMappings = false;
	public QuestUnfolder(NativeQueryLanguageComponentFactory nativeQLFactory,
						 IMapping2DatalogConverter mapping2DatalogConvertor,
						 QuestCoreSettings preferences) throws Exception{
		this.nativeQLFactory = nativeQLFactory;
		this.mapping2DatalogConvertor = mapping2DatalogConvertor;
		this.preferences = preferences;
	}


    /**
	 * Setting up the unfolder and SQL generation
	 */
	public void setupInVirtualMode(Collection<OBDAMappingAxiom> mappingAxioms, DBMetadata metadata,
								   DBConnector dbConnector,
								   VocabularyValidator vocabularyValidator, TBoxReasoner reformulationReasoner,
								   Ontology inputOntology, TMappingExclusionConfig excludeFromTMappings)
			throws SQLException, JSQLParserException, OBDAException {

		mappingAxioms = vocabularyValidator.replaceEquivalences(mappingAxioms);

		Collection<OBDAMappingAxiom> normalizedMappingAxioms = dbConnector.applyDBSpecificNormalization(mappingAxioms,
				metadata);

		/*
         * add sameAsInverse
         */
		if (preferences.isSameAsInMappingsEnabled()) {
			normalizedMappingAxioms = MappingSameAs.addSameAsInverse(normalizedMappingAxioms, nativeQLFactory);
		}


		List<CQIE> unfoldingProgram = mapping2DatalogConvertor.constructDatalogProgram(normalizedMappingAxioms, metadata);


		log.debug("Original mapping size: {}", unfoldingProgram.size());

		// Normalizing language tags and equalities
		normalizeMappings(unfoldingProgram);

		// Apply TMappings
		unfoldingProgram = applyTMappings(unfoldingProgram, reformulationReasoner, true, metadata, excludeFromTMappings);

		// Adding ontology assertions (ABox) as rules (facts, head with no body).
		addAssertionsAsFacts(unfoldingProgram, inputOntology.getClassAssertions(),
				inputOntology.getObjectPropertyAssertions(), inputOntology.getDataPropertyAssertions(),
				inputOntology.getAnnotationAssertions());

		// Adding data typing on the mapping axioms.
		 // Adding NOT NULL conditions to the variables used in the head
		 // of all mappings to preserve SQL-RDF semantics
		extendTypesWithMetadata(unfoldingProgram, reformulationReasoner, vocabularyValidator, metadata);
		addNOTNULLToMappings(unfoldingProgram, metadata);

		// Adding ontology assertions (ABox) as rules (facts, head with no body).
		List<AnnotationAssertion> annotationAssertions;
		if (preferences.isOntologyAnnotationQueryingEnabled()) {
			annotationAssertions = inputOntology.getAnnotationAssertions();
		}
		else{
			annotationAssertions = Collections.emptyList();
		}

		// Temporary (needed by the assertions)
		uriTemplateMatcher = UriTemplateMatcher.create(unfoldingProgram);

		addAssertionsAsFacts(unfoldingProgram, inputOntology.getClassAssertions(),
				inputOntology.getObjectPropertyAssertions(), inputOntology.getDataPropertyAssertions(), annotationAssertions);

		if (preferences.isSameAsInMappingsEnabled()) {
			addSameAsMapping(unfoldingProgram);
		}

        if(log.isDebugEnabled()) {
            String finalMappings = Joiner.on("\n").join(unfoldingProgram);
            log.debug("Set of mappings before canonical IRI rewriting: \n {}", finalMappings);<commit_msg>remove duplicate call to annotation assertion<commit_after>package it.unibz.inf.ontop.owlrefplatform.core;


import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
import it.unibz.inf.ontop.injection.NativeQueryLanguageComponentFactory;
import it.unibz.inf.ontop.injection.QuestCoreSettings;
import it.unibz.inf.ontop.model.*;
import it.unibz.inf.ontop.model.impl.OBDAVocabulary;
import it.unibz.inf.ontop.model.impl.TermUtils;
import it.unibz.inf.ontop.ontology.*;
import it.unibz.inf.ontop.owlrefplatform.core.basicoperations.CQContainmentCheckUnderLIDs;
import it.unibz.inf.ontop.owlrefplatform.core.basicoperations.EQNormalizer;
import it.unibz.inf.ontop.owlrefplatform.core.basicoperations.LinearInclusionDependencies;
import it.unibz.inf.ontop.owlrefplatform.core.basicoperations.VocabularyValidator;
import it.unibz.inf.ontop.owlrefplatform.core.dagjgrapht.TBoxReasoner;
import it.unibz.inf.ontop.owlrefplatform.core.mappingprocessing.*;
import it.unibz.inf.ontop.owlrefplatform.core.unfolding.DatalogUnfolder;
import it.unibz.inf.ontop.pivotalrepr.MetadataForQueryOptimization;
import it.unibz.inf.ontop.pivotalrepr.impl.MetadataForQueryOptimizationImpl;
import it.unibz.inf.ontop.sql.DatabaseRelationDefinition;
import it.unibz.inf.ontop.sql.RDBMetadata;
import it.unibz.inf.ontop.sql.Relation2DatalogPredicate;
import it.unibz.inf.ontop.sql.RelationID;
import it.unibz.inf.ontop.utils.IMapping2DatalogConverter;
import it.unibz.inf.ontop.utils.ImmutableCollectors;
import net.sf.jsqlparser.JSQLParserException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.SQLException;
import java.util.*;
import java.util.stream.IntStream;

import static it.unibz.inf.ontop.model.impl.OntopModelSingletons.DATA_FACTORY;

public class QuestUnfolder {

	private final NativeQueryLanguageComponentFactory nativeQLFactory;
	private final QuestCoreSettings preferences;
	/* The active unfolding engine */
	private DatalogUnfolder unfolder;

	/*
	 * These are pattern matchers that will help transforming the URI's in
	 * queries into Functions, used by the SPARQL translator.
	 */
	private UriTemplateMatcher uriTemplateMatcher = new UriTemplateMatcher();

	protected List<CQIE> ufp; // for TESTS ONLY

	private static final Logger log = LoggerFactory.getLogger(QuestUnfolder.class);

	private ImmutableMultimap<AtomPredicate, ImmutableList<Integer>> uniqueConstraints;
	private final IMapping2DatalogConverter mapping2DatalogConvertor;
	private MetadataForQueryOptimization metadataForQueryOptimization;

	private Set<Predicate> dataPropertiesAndClassesMapped = new HashSet<>();
	private Set<Predicate> objectPropertiesMapped = new HashSet<>();

	/** Davide> Whether to exclude the user-supplied predicates from the
	 *          TMapping procedure (that is, the mapping assertions for
	 *          those predicates should not be extended according to the
	 *          TBox hierarchies
	 */
	//private boolean applyExcludeFromTMappings = false;
	public QuestUnfolder(NativeQueryLanguageComponentFactory nativeQLFactory,
						 IMapping2DatalogConverter mapping2DatalogConvertor,
						 QuestCoreSettings preferences) throws Exception{
		this.nativeQLFactory = nativeQLFactory;
		this.mapping2DatalogConvertor = mapping2DatalogConvertor;
		this.preferences = preferences;
	}


    /**
	 * Setting up the unfolder and SQL generation
	 */
	public void setupInVirtualMode(Collection<OBDAMappingAxiom> mappingAxioms, DBMetadata metadata,
								   DBConnector dbConnector,
								   VocabularyValidator vocabularyValidator, TBoxReasoner reformulationReasoner,
								   Ontology inputOntology, TMappingExclusionConfig excludeFromTMappings)
			throws SQLException, JSQLParserException, OBDAException {

		mappingAxioms = vocabularyValidator.replaceEquivalences(mappingAxioms);

		Collection<OBDAMappingAxiom> normalizedMappingAxioms = dbConnector.applyDBSpecificNormalization(mappingAxioms,
				metadata);

		/*
         * add sameAsInverse
         */
		if (preferences.isSameAsInMappingsEnabled()) {
			normalizedMappingAxioms = MappingSameAs.addSameAsInverse(normalizedMappingAxioms, nativeQLFactory);
		}


		List<CQIE> unfoldingProgram = mapping2DatalogConvertor.constructDatalogProgram(normalizedMappingAxioms, metadata);


		log.debug("Original mapping size: {}", unfoldingProgram.size());

		// Normalizing language tags and equalities
		normalizeMappings(unfoldingProgram);

		// Apply TMappings
		unfoldingProgram = applyTMappings(unfoldingProgram, reformulationReasoner, true, metadata, excludeFromTMappings);


		// Adding data typing on the mapping axioms.
		 // Adding NOT NULL conditions to the variables used in the head
		 // of all mappings to preserve SQL-RDF semantics
		extendTypesWithMetadata(unfoldingProgram, reformulationReasoner, vocabularyValidator, metadata);
		addNOTNULLToMappings(unfoldingProgram, metadata);

		// Adding ontology assertions (ABox) as rules (facts, head with no body).
		List<AnnotationAssertion> annotationAssertions;
		if (preferences.isOntologyAnnotationQueryingEnabled()) {
			annotationAssertions = inputOntology.getAnnotationAssertions();
		}
		else{
			annotationAssertions = Collections.emptyList();
		}

		// Temporary (needed by the assertions)
		uriTemplateMatcher = UriTemplateMatcher.create(unfoldingProgram);

		// Adding ontology assertions (ABox) as rules (facts, head with no body).
		addAssertionsAsFacts(unfoldingProgram, inputOntology.getClassAssertions(),
				inputOntology.getObjectPropertyAssertions(), inputOntology.getDataPropertyAssertions(), annotationAssertions);

		if (preferences.isSameAsInMappingsEnabled()) {
			addSameAsMapping(unfoldingProgram);
		}

        if(log.isDebugEnabled()) {
            String finalMappings = Joiner.on("\n").join(unfoldingProgram);
            log.debug("Set of mappings before canonical IRI rewriting: \n {}", finalMappings);
{'fraction_non_alphanumeric': 0.08301542544368884, 'fraction_numerical': 0.0018245148449162383, 'mean_word_length': 5.625824175824176, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4446195', 'n_tokens_mistral': 3812, 'n_tokens_neox': 3509, 'n_words': 687}