<commit_before>/**
 * 
 */
package com.github.podd.api.test;

import java.io.InputStream;
import java.util.Collections;
import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.openrdf.model.Graph;
import org.openrdf.model.Statement;
import org.openrdf.model.URI;
import org.openrdf.model.impl.GraphImpl;
import org.openrdf.model.vocabulary.RDF;
import org.openrdf.query.GraphQuery;
import org.openrdf.query.GraphQueryResult;
import org.openrdf.query.QueryLanguage;
import org.openrdf.query.impl.DatasetImpl;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.Rio;
import org.semanticweb.owlapi.model.OWLOntologyID;

import com.github.podd.api.PoddArtifactManager;
import com.github.podd.api.PoddProcessorStage;
import com.github.podd.api.file.PoddFileReference;
import com.github.podd.api.file.PoddFileReferenceManager;
import com.github.podd.api.file.PoddFileReferenceProcessor;
import com.github.podd.api.file.PoddFileReferenceProcessorFactory;
import com.github.podd.api.file.PoddFileReferenceProcessorFactoryRegistry;
import com.github.podd.api.purl.PoddPurlManager;
import com.github.podd.api.purl.PoddPurlProcessor;
import com.github.podd.api.purl.PoddPurlProcessorFactory;
import com.github.podd.api.purl.PoddPurlProcessorFactoryRegistry;
import com.github.podd.api.purl.PoddPurlReference;
import com.github.podd.utils.InferredOWLOntologyID;

/**
 * @author <NAME> <EMAIL>
 * 
 */
public abstract class AbstractPoddArtifactManagerTest
{
    protected abstract PoddArtifactManager getNewArtifactManager();
    
    protected abstract PoddPurlProcessorFactory getNewDoiPurlProcessorFactory();
    
    protected abstract PoddFileReferenceManager getNewFileReferenceManager();
    
    protected abstract PoddPurlProcessorFactory getNewHandlePurlProcessorFactory();
    
    protected abstract PoddFileReferenceProcessorFactory getNewHttpFileReferenceProcessorFactory();
    
    protected abstract PoddPurlManager getNewPurlManager();
    
    protected abstract PoddFileReferenceProcessorFactory getNewSSHFileReferenceProcessorFactory();
    
    protected abstract PoddPurlProcessorFactory getNewUUIDPurlProcessorFactory();
    
    /**
     * @throws java.lang.Exception
     */
    @SuppressWarnings("null")
    @Before
    public void setUp() throws Exception
    {
        final PoddFileReferenceProcessorFactoryRegistry testFileRegistry =
                new PoddFileReferenceProcessorFactoryRegistry();
        // clear any automatically added entries that may come from META-INF/services entries on the
        // classpath
        testFileRegistry.clear();
        // In practice, the following factories would be automatically added to the registry,
        // however for testing we want to explicitly add the ones we want to support for each test
        testFileRegistry.add(this.getNewSSHFileReferenceProcessorFactory());
        testFileRegistry.add(this.getNewHttpFileReferenceProcessorFactory());
        
        final PoddPurlProcessorFactoryRegistry testPurlRegistry = new PoddPurlProcessorFactoryRegistry();
        testPurlRegistry.clear();
        testPurlRegistry.add(this.getNewDoiPurlProcessorFactory());
        testPurlRegistry.add(this.getNewHandlePurlProcessorFactory());
        testPurlRegistry.add(this.getNewUUIDPurlProcessorFactory());
        
        final PoddFileReferenceManager testFileReferenceManager = this.getNewFileReferenceManager();
        testFileReferenceManager.setProcessorFactoryRegistry(testFileRegistry);
        
        final PoddPurlManager testPurlManager = this.getNewPurlManager();
        testPurlManager.setPurlFactoryRegistry(testPurlRegistry);
        
        final PoddArtifactManager testArtifactManager = this.getNewArtifactManager();
        testArtifactManager.setFileReferenceManager(testFileReferenceManager);
        testArtifactManager.setPurlManager(testPurlManager);
        
        final InputStream inputStream = this.getClass().getResourceAsStream("/testArtifact.rdf");
        // MIME type should be either given by the user, detected from the content type on the
        // request, or autodetected using the Any23 Mime Detector
        final String mimeType = "application/rdf+xml";
        final RDFFormat format = Rio.getParserFormatForMIMEType(mimeType, RDFFormat.RDFXML);
        
        final InferredOWLOntologyID resultArtifactId = testArtifactManager.loadArtifact(inputStream, format);
        
        // INSIDE the loadArtifact method...
        
        // testArtifactManager needs to create artifactId before attempting to extract file
        // references
        final OWLOntologyID tempArtifactId = null;
        
        // connection to the repository that the artifact RDF triples will be stored in
        final RepositoryConnection tempConn = null;
        
        // TODO: Load the artifact RDF triples into a specific context in the temp repository
        tempConn.add(inputStream, "", format);
        
        // TODO: Identify the Artifact IRI and Artifact Version IRI
        
        // return the results, setting the results variable to be the same as the internalResults
        // variable from inside of extractFileReferences
        // Ie, return internalResults; results = internalResults;
        
        final Set<PoddPurlReference> purlResults =
                testArtifactManager.getPurlManager().extractPurlReferences(tempConn,
                        tempArtifactId.getVersionIRI().toOpenRDFURI());
        
        // INSIDE extractPurlReferences
        final Set<PoddPurlReference> internalPurlResults =
                Collections.newSetFromMap(new ConcurrentHashMap<PoddPurlReference, Boolean>());
        
        for(final PoddPurlProcessorFactory nextProcessorFactory : testArtifactManager.getPurlManager()
                .getProcessorFactoryRegistry().getByStage(PoddProcessorStage.RDF_PARSING))
        {
            final StringBuilder sparqlQuery = new StringBuilder();
            sparqlQuery.append("CONSTRUCT { ");
            sparqlQuery.append(nextProcessorFactory.getSPARQLConstructBGP());
            sparqlQuery.append(" } WHERE { ");
            sparqlQuery.append(nextProcessorFactory.getSPARQLConstructWhere());
            sparqlQuery.append(" } ");
            if(!nextProcessorFactory.getSPARQLGroupBy().isEmpty())
            {
                sparqlQuery.append(" GROUP BY ");
                sparqlQuery.append(nextProcessorFactory.getSPARQLGroupBy());
            }
            
            final GraphQuery graphQuery = tempConn.prepareGraphQuery(QueryLanguage.SPARQL, sparqlQuery.toString());
            
            // Create a new dataset to specify the contexts that the query will be allowed to access
            final DatasetImpl dataset = new DatasetImpl();
            // The following URIs are passed in as the context to the
            // extractFileReferences(RepositoryConnection,URI...) method
            final URI artifactGraphUri = null;
            
            // for(URI artifactGraphUri : contexts)
            dataset.addDefaultGraph(artifactGraphUri);
            dataset.addNamedGraph(artifactGraphUri);
            // end for loop
            
            // if the stage is after inferencing, the inferred graph URI would be one of the
            // contexts as well
            final URI artifactInferredGraphUri = null;
            dataset.addDefaultGraph(artifactInferredGraphUri);
            dataset.addNamedGraph(artifactInferredGraphUri);
            
            // set the dataset for the query to be our artificially constructed dataset
            graphQuery.setDataset(dataset);
            
            final GraphQueryResult queryResult = graphQuery.evaluate();
            
            // If the query matched anything, then for each of the file references in the resulting
            // construct statements, we create a file reference and add it to the results
            if(!queryResult.hasNext())
            {
                while(queryResult.hasNext())
                {
                    final Statement next = queryResult.next();
                    // This processor factory matches the graph that we wish to use, so we create a
                    // processor instance now to create the file reference
                    // NOTE: This object cannot be shared as we do not specify that it needs to be
                    // threadsafe
                    final PoddPurlProcessor processor = nextProcessorFactory.getProcessor();
                    
                    if(next.getSubject() instanceof URI && processor.canHandle((URI)next.getSubject()))
                    {
                        internalPurlResults.add(processor.handleTranslation((URI)next.getSubject()));
                    }
                }
            }
        }<commit_msg>avoid duplicate calls to purl processors for a given temporary URI in any RDF statements uploaded to the podd artifact manager<commit_after>/**
 * 
 */
package com.github.podd.api.test;

import java.io.InputStream;
import java.util.Collections;
import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.openrdf.model.Graph;
import org.openrdf.model.Statement;
import org.openrdf.model.URI;
import org.openrdf.model.impl.GraphImpl;
import org.openrdf.model.vocabulary.RDF;
import org.openrdf.query.GraphQuery;
import org.openrdf.query.GraphQueryResult;
import org.openrdf.query.QueryLanguage;
import org.openrdf.query.impl.DatasetImpl;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.Rio;
import org.semanticweb.owlapi.model.OWLOntologyID;

import com.github.podd.api.PoddArtifactManager;
import com.github.podd.api.PoddProcessorStage;
import com.github.podd.api.file.PoddFileReference;
import com.github.podd.api.file.PoddFileReferenceManager;
import com.github.podd.api.file.PoddFileReferenceProcessor;
import com.github.podd.api.file.PoddFileReferenceProcessorFactory;
import com.github.podd.api.file.PoddFileReferenceProcessorFactoryRegistry;
import com.github.podd.api.purl.PoddPurlManager;
import com.github.podd.api.purl.PoddPurlProcessor;
import com.github.podd.api.purl.PoddPurlProcessorFactory;
import com.github.podd.api.purl.PoddPurlProcessorFactoryRegistry;
import com.github.podd.api.purl.PoddPurlReference;
import com.github.podd.utils.InferredOWLOntologyID;

/**
 * @author <NAME> <EMAIL>
 * 
 */
public abstract class AbstractPoddArtifactManagerTest
{
    protected abstract PoddArtifactManager getNewArtifactManager();
    
    protected abstract PoddPurlProcessorFactory getNewDoiPurlProcessorFactory();
    
    protected abstract PoddFileReferenceManager getNewFileReferenceManager();
    
    protected abstract PoddPurlProcessorFactory getNewHandlePurlProcessorFactory();
    
    protected abstract PoddFileReferenceProcessorFactory getNewHttpFileReferenceProcessorFactory();
    
    protected abstract PoddPurlManager getNewPurlManager();
    
    protected abstract PoddFileReferenceProcessorFactory getNewSSHFileReferenceProcessorFactory();
    
    protected abstract PoddPurlProcessorFactory getNewUUIDPurlProcessorFactory();
    
    /**
     * @throws java.lang.Exception
     */
    @SuppressWarnings("null")
    @Before
    public void setUp() throws Exception
    {
        final PoddFileReferenceProcessorFactoryRegistry testFileRegistry =
                new PoddFileReferenceProcessorFactoryRegistry();
        // clear any automatically added entries that may come from META-INF/services entries on the
        // classpath
        testFileRegistry.clear();
        // In practice, the following factories would be automatically added to the registry,
        // however for testing we want to explicitly add the ones we want to support for each test
        testFileRegistry.add(this.getNewSSHFileReferenceProcessorFactory());
        testFileRegistry.add(this.getNewHttpFileReferenceProcessorFactory());
        
        final PoddPurlProcessorFactoryRegistry testPurlRegistry = new PoddPurlProcessorFactoryRegistry();
        testPurlRegistry.clear();
        testPurlRegistry.add(this.getNewDoiPurlProcessorFactory());
        testPurlRegistry.add(this.getNewHandlePurlProcessorFactory());
        testPurlRegistry.add(this.getNewUUIDPurlProcessorFactory());
        
        final PoddFileReferenceManager testFileReferenceManager = this.getNewFileReferenceManager();
        testFileReferenceManager.setProcessorFactoryRegistry(testFileRegistry);
        
        final PoddPurlManager testPurlManager = this.getNewPurlManager();
        testPurlManager.setPurlFactoryRegistry(testPurlRegistry);
        
        final PoddArtifactManager testArtifactManager = this.getNewArtifactManager();
        testArtifactManager.setFileReferenceManager(testFileReferenceManager);
        testArtifactManager.setPurlManager(testPurlManager);
        
        final InputStream inputStream = this.getClass().getResourceAsStream("/testArtifact.rdf");
        // MIME type should be either given by the user, detected from the content type on the
        // request, or autodetected using the Any23 Mime Detector
        final String mimeType = "application/rdf+xml";
        final RDFFormat format = Rio.getParserFormatForMIMEType(mimeType, RDFFormat.RDFXML);
        
        final InferredOWLOntologyID resultArtifactId = testArtifactManager.loadArtifact(inputStream, format);
        
        // INSIDE the loadArtifact method...
        
        // testArtifactManager needs to create artifactId before attempting to extract file
        // references
        final OWLOntologyID tempArtifactId = null;
        
        // connection to the repository that the artifact RDF triples will be stored in
        final RepositoryConnection tempConn = null;
        
        // TODO: Load the artifact RDF triples into a specific context in the temp repository
        tempConn.add(inputStream, "", format);
        
        // TODO: Identify the Artifact IRI and Artifact Version IRI
        
        // return the results, setting the results variable to be the same as the internalResults
        // variable from inside of extractFileReferences
        // Ie, return internalResults; results = internalResults;
        
        final Set<PoddPurlReference> purlResults =
                testArtifactManager.getPurlManager().extractPurlReferences(tempConn,
                        tempArtifactId.getVersionIRI().toOpenRDFURI());
        
        // INSIDE extractPurlReferences
        final Set<PoddPurlReference> internalPurlResults =
                Collections.newSetFromMap(new ConcurrentHashMap<PoddPurlReference, Boolean>());
        final Set<URI> temporaryURIs = Collections.newSetFromMap(new ConcurrentHashMap<URI, Boolean>());
        for(final PoddPurlProcessorFactory nextProcessorFactory : testArtifactManager.getPurlManager()
                .getProcessorFactoryRegistry().getByStage(PoddProcessorStage.RDF_PARSING))
        {
            final StringBuilder sparqlQuery = new StringBuilder();
            sparqlQuery.append("CONSTRUCT { ");
            sparqlQuery.append(nextProcessorFactory.getSPARQLConstructBGP());
            sparqlQuery.append(" } WHERE { ");
            sparqlQuery.append(nextProcessorFactory.getSPARQLConstructWhere());
            sparqlQuery.append(" } ");
            if(!nextProcessorFactory.getSPARQLGroupBy().isEmpty())
            {
                sparqlQuery.append(" GROUP BY ");
                sparqlQuery.append(nextProcessorFactory.getSPARQLGroupBy());
            }
            
            final GraphQuery graphQuery = tempConn.prepareGraphQuery(QueryLanguage.SPARQL, sparqlQuery.toString());
            
            // Create a new dataset to specify the contexts that the query will be allowed to access
            final DatasetImpl dataset = new DatasetImpl();
            // The following URIs are passed in as the context to the
            // extractFileReferences(RepositoryConnection,URI...) method
            final URI artifactGraphUri = null;
            
            // for(URI artifactGraphUri : contexts)
            dataset.addDefaultGraph(artifactGraphUri);
            dataset.addNamedGraph(artifactGraphUri);
            // end for loop
            
            // if the stage is after inferencing, the inferred graph URI would be one of the
            // contexts as well
            final URI artifactInferredGraphUri = null;
            dataset.addDefaultGraph(artifactInferredGraphUri);
            dataset.addNamedGraph(artifactInferredGraphUri);
            
            // set the dataset for the query to be our artificially constructed dataset
            graphQuery.setDataset(dataset);
            
            final GraphQueryResult queryResult = graphQuery.evaluate();
            
            // If the query matched anything, then for each of the file references in the resulting
            // construct statements, we create a file reference and add it to the results
            if(!queryResult.hasNext())
            {
                while(queryResult.hasNext())
                {
                    final Statement next = queryResult.next();
                    // This processor factory matches the graph that we wish to use, so we create a
                    // processor instance now to create the file reference
                    // NOTE: This object cannot be shared as we do not specify that it needs to be
                    // threadsafe
                    final PoddPurlProcessor processor = nextProcessorFactory.getProcessor();
                    
                    // NOTE: We use a Set to avoid duplicate calls to any purl processors for any
                    // temporary URI
                    if(next.getSubject() instanceof URI && !temporaryURIs.contains(next.getSubject())
                            && processor.canHandle((URI)next.getSubject()))
                    {
                        temporaryURIs.add((URI)next.getSubject());
                        internalPurlResults.add(processor.handleTranslation((URI)next.getSubject()));
                    }
                    
                    // NOTE: Predicate rewriting is not supported. Predicates in OWL Documents must
                    // be URIs from recognise vocabularies, so cannot be autogenerated PURLs
                    
                    if(next.getObject() instanceof URI && !temporaryURIs.contains(next.getObject())
                            && processor.canHandle((URI)next.getObject()))
                    {
                        temporaryURIs.add((URI)next.getObject());
                        internalPurlResults.add(processor.handleTranslation((URI)next.getObject()));
                    }
                }
            }
        }
{'fraction_non_alphanumeric': 0.07257388388178579, 'fraction_numerical': 0.00020959966464053657, 'mean_word_length': 2.719547846423699, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7664162', 'n_tokens_mistral': 4827, 'n_tokens_neox': 4723, 'n_words': 1142}