<commit_before>import hashlib
from io import BytesIO
import mimetypes
import os
import pprint
import requests
import rdflib
from rdflib import Namespace

#============================================================================
# NAMESPACE BINDINGS
#============================================================================

namespace_manager = rdflib.namespace.NamespaceManager(rdflib.Graph())

bibo = Namespace('http://purl.org/ontology/bibo/')
namespace_manager.bind('bibo', bibo, override=False)

dc = Namespace('http://purl.org/dc/elements/1.1/')
namespace_manager.bind('dc', dc, override=False)

ex = Namespace('http://www.example.org/terms/')
namespace_manager.bind('ex', ex, override=False)

foaf = Namespace('http://xmlns.com/foaf/0.1/')
namespace_manager.bind('foaf', foaf, override=False)

iana = Namespace('http://www.iana.org/assignments/relation/')
namespace_manager.bind('iana', iana, override=False)

ore = Namespace('http://www.openarchives.org/ore/terms/')
namespace_manager.bind('ore', ore, override=False)

pcdm = Namespace('http://pcdm.org/models#')
namespace_manager.bind('pcdm', pcdm, override=False)

rdf = Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')
namespace_manager.bind('rdf', rdf, override=False)



#============================================================================
# REPOSITORY (REPRESENTING AN FCREPO INSTANCE)
#============================================================================

class Repository():
    def __init__(self, config):
        self.endpoint = config['REST_ENDPOINT']
        if 'FEDORA_USER' in config and 'FEDORA_PASSWORD' in config:
            self.auth = (config['FEDORA_USER'], config['FEDORA_PASSWORD'])
        else:
            self.auth = ()

    def post(self, url, **kwargs):
        return requests.post(url, auth=self.auth, **kwargs)

    def patch(self, url, **kwargs):
        return requests.patch(url, auth=self.auth, **kwargs)

    def head(self, url, **kwargs):
        return requests.head(url, auth=self.auth, **kwargs)


#============================================================================
# PCDM RESOURCE (COMMON METHODS FOR ALL OBJECTS)
#============================================================================

class Resource():

    def __init__(self, uri=''):
        self.graph = rdflib.Graph()
        self.graph.namespace_manager = namespace_manager
        self.uri = rdflib.URIRef(uri)


    # create repository object by POSTing object graph
    def create_object(self, repository):
        if self.exists_in_repo(repository):
            return False
        else:
            print("Creating {0}...".format(self.title), end='')
            response = repository.post(repository.endpoint)
            if response.status_code == 201:
                print("success.")
                print(response.status_code, response.text)
                self.uri = rdflib.URIRef(response.text)
                return True
            else:
                print("failed!")
                return False


    # update existing repo object with SPARQL update
    def update_object(self, repository):
        print("Patching {0}...".format(str(self.uri)), end='')
        prolog = ''
        #TODO: limit this to just the prefixes that are used in the graph
        for (prefix, uri) in self.graph.namespace_manager.namespaces():
            prolog += "PREFIX {0}: {1}\n".format(prefix, uri.n3())

        triples = [ "<> {0} {1}.".format(
            self.graph.namespace_manager.normalizeUri(p),
            o.n3()) for (s, p, o) in self.graph ]

        query = prolog + "INSERT DATA {{{0}}}".format("\n".join(triples))
        data = query.encode('utf-8')
        headers = {'Content-Type': 'application/sparql-update'}
        response = repository.patch(str(self.uri), data=data, headers=headers)
        if response.status_code == 204:
            print("success.")
        else:
            print("failed!")
            print(response.status_code, response.text)
        return response


    # recursively create an object and components and that don't yet exist
    def recursive_create(self, repository, nobinaries):
        if not self.exists_in_repo(repository):
            self.create_object(repository)
        else:
            print('Object "{0}" exists. Skipping...'.format(self.title))

        if not nobinaries:
            for file in self.files:
                if not file.exists_in_repo(repository):
                    file.create_nonrdf(repository)
                else:
                    print('File "{0}" exists. Skipping...'.format(file.title))

        for component in self.components:
            if not component.exists_in_repo(repository):
                component.recursive_create(repository, nobinaries)
            else:
                print(
                    'Component "{0}" exists. Skipping...'.format(
                        component.title)
                    )

        if hasattr(self, 'collections'):
            for collection in self.collections:
                if not collection.exists_in_repo(repository):
                    collection.create_object(repository)
                else:
                    print(
                        'Collection "{0}" exists. Skipping...'.format(
                            collection.title)
                        )


    # recursively update an object and all its components and files
    def recursive_update(self, repository, nobinaries):
        self.update_object(repository)
        if not nobinaries:
            for file in self.files:
                file.update_object(repository)
        for component in self.components:
            component.recursive_update(repository, nobinaries)
        if hasattr(self, 'collections'):
            for collection in self.collections:
                collection.update_object(repository)


    # check for the existence of a local object in the repository
    def exists_in_repo(self, repository):
        if str(self.uri).startswith(repository.endpoint):
            response = repository.head(str(self.uri))
            if response.status_code == 200:
                return True
            else:
                return False
        else:
            return False


    # update object graph with URI
    def update_subject_uri(self):
        for (s, p, o) in self.graph:
            self.graph.delete( (s, p, o) )
            self.graph.add( (self.uri, p, o) )


    # update membership triples (can be used after repository object creation
    # to ensure repository URIs are present)
    def update_relationship_triples(self):
        for component in self.components:
            self.graph.add( (self.uri, pcdm.hasMember, component.uri) )
            component.graph.add( (component.uri, pcdm.memberOf, self.uri) )

        for file in self.files:
            self.graph.add( (self.uri, pcdm.hasFile, file.uri) )
            file.graph.add( (file.uri, pcdm.fileOf, self.uri) )

        for collection in self.collections:
            self.graph.add( (self.uri, pcdm.memberOf, collection.uri) )
            collection.graph.add( (collection.uri, pcdm.hasMember, self.uri) )


    # show the object's graph, serialized as turtle
    def print_graph(self):
        print(self.graph.serialize(format="turtle").decode())


    # show the item graph and tree of related objects<commit_msg>LIBFCREPO-114. Client cert authentication.<commit_after>import hashlib
from io import BytesIO
import mimetypes
import os
import pprint
import requests
import rdflib
from rdflib import Namespace

#============================================================================
# NAMESPACE BINDINGS
#============================================================================

namespace_manager = rdflib.namespace.NamespaceManager(rdflib.Graph())

bibo = Namespace('http://purl.org/ontology/bibo/')
namespace_manager.bind('bibo', bibo, override=False)

dc = Namespace('http://purl.org/dc/elements/1.1/')
namespace_manager.bind('dc', dc, override=False)

ex = Namespace('http://www.example.org/terms/')
namespace_manager.bind('ex', ex, override=False)

foaf = Namespace('http://xmlns.com/foaf/0.1/')
namespace_manager.bind('foaf', foaf, override=False)

iana = Namespace('http://www.iana.org/assignments/relation/')
namespace_manager.bind('iana', iana, override=False)

ore = Namespace('http://www.openarchives.org/ore/terms/')
namespace_manager.bind('ore', ore, override=False)

pcdm = Namespace('http://pcdm.org/models#')
namespace_manager.bind('pcdm', pcdm, override=False)

rdf = Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')
namespace_manager.bind('rdf', rdf, override=False)



#============================================================================
# REPOSITORY (REPRESENTING AN FCREPO INSTANCE)
#============================================================================

class Repository():
    def __init__(self, config):
        self.endpoint = config['REST_ENDPOINT']
        self.auth = None
        self.client_cert = None

        if 'CLIENT_CERT' in config and 'CLIENT_KEY' in config:
            self.client_cert = (config['CLIENT_CERT'], config['CLIENT_KEY'])
        elif 'FEDORA_USER' in config and 'FEDORA_PASSWORD' in config:
            self.auth = (config['FEDORA_USER'], config['FEDORA_PASSWORD'])

    def post(self, url, **kwargs):
        return requests.post(url, cert=self.client_cert, auth=self.auth, **kwargs)

    def patch(self, url, **kwargs):
        return requests.patch(url, cert=self.client_cert, auth=self.auth, **kwargs)

    def head(self, url, **kwargs):
        return requests.head(url, cert=self.client_cert, auth=self.auth, **kwargs)


#============================================================================
# PCDM RESOURCE (COMMON METHODS FOR ALL OBJECTS)
#============================================================================

class Resource():

    def __init__(self, uri=''):
        self.graph = rdflib.Graph()
        self.graph.namespace_manager = namespace_manager
        self.uri = rdflib.URIRef(uri)


    # create repository object by POSTing object graph
    def create_object(self, repository):
        if self.exists_in_repo(repository):
            return False
        else:
            print("Creating {0}...".format(self.title), end='')
            response = repository.post(repository.endpoint)
            if response.status_code == 201:
                print("success.")
                print(response.status_code, response.text)
                self.uri = rdflib.URIRef(response.text)
                return True
            else:
                print("failed!")
                return False


    # update existing repo object with SPARQL update
    def update_object(self, repository):
        print("Patching {0}...".format(str(self.uri)), end='')
        prolog = ''
        #TODO: limit this to just the prefixes that are used in the graph
        for (prefix, uri) in self.graph.namespace_manager.namespaces():
            prolog += "PREFIX {0}: {1}\n".format(prefix, uri.n3())

        triples = [ "<> {0} {1}.".format(
            self.graph.namespace_manager.normalizeUri(p),
            o.n3()) for (s, p, o) in self.graph ]

        query = prolog + "INSERT DATA {{{0}}}".format("\n".join(triples))
        data = query.encode('utf-8')
        headers = {'Content-Type': 'application/sparql-update'}
        response = repository.patch(str(self.uri), data=data, headers=headers)
        if response.status_code == 204:
            print("success.")
        else:
            print("failed!")
            print(response.status_code, response.text)
        return response


    # recursively create an object and components and that don't yet exist
    def recursive_create(self, repository, nobinaries):
        if not self.exists_in_repo(repository):
            self.create_object(repository)
        else:
            print('Object "{0}" exists. Skipping...'.format(self.title))

        if not nobinaries:
            for file in self.files:
                if not file.exists_in_repo(repository):
                    file.create_nonrdf(repository)
                else:
                    print('File "{0}" exists. Skipping...'.format(file.title))

        for component in self.components:
            if not component.exists_in_repo(repository):
                component.recursive_create(repository, nobinaries)
            else:
                print(
                    'Component "{0}" exists. Skipping...'.format(
                        component.title)
                    )

        if hasattr(self, 'collections'):
            for collection in self.collections:
                if not collection.exists_in_repo(repository):
                    collection.create_object(repository)
                else:
                    print(
                        'Collection "{0}" exists. Skipping...'.format(
                            collection.title)
                        )


    # recursively update an object and all its components and files
    def recursive_update(self, repository, nobinaries):
        self.update_object(repository)
        if not nobinaries:
            for file in self.files:
                file.update_object(repository)
        for component in self.components:
            component.recursive_update(repository, nobinaries)
        if hasattr(self, 'collections'):
            for collection in self.collections:
                collection.update_object(repository)


    # check for the existence of a local object in the repository
    def exists_in_repo(self, repository):
        if str(self.uri).startswith(repository.endpoint):
            response = repository.head(str(self.uri))
            if response.status_code == 200:
                return True
            else:
                return False
        else:
            return False


    # update object graph with URI
    def update_subject_uri(self):
        for (s, p, o) in self.graph:
            self.graph.delete( (s, p, o) )
            self.graph.add( (self.uri, p, o) )


    # update membership triples (can be used after repository object creation
    # to ensure repository URIs are present)
    def update_relationship_triples(self):
        for component in self.components:
            self.graph.add( (self.uri, pcdm.hasMember, component.uri) )
            component.graph.add( (component.uri, pcdm.memberOf, self.uri) )

        for file in self.files:
            sys.stderr.write(str(self.uri))
            self.graph.add( (self.uri, pcdm.hasFile, file.uri) )
            file.graph.add( (file.uri, pcdm.fileOf, self.uri) )

        for collection in self.collections:
            self.graph.add( (self.uri, pcdm.memberOf, collection.uri) )
            collection.graph.add( (collection.uri, pcdm.hasMember, self.uri) )


    # show the object's graph, serialized as turtle
    def print_graph(self):
        print(self.graph.serialize(format="turtle").decode())


    # show the item graph and tree of related objects
{'fraction_non_alphanumeric': 0.17096065508288397, 'fraction_numerical': 0.004993009786299181, 'mean_word_length': 2.5496219281663515, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2705476', 'n_tokens_mistral': 4256, 'n_tokens_neox': 4020, 'n_words': 961}