<commit_before>import RDF
from base64 import standard_b64encode as b64encode
from soaplib.client import make_service_client
from soaplib.serializers import primitive as soap_types
from soaplib.service import soapmethod
from soaplib.wsgi_soap import SimpleWSGISoapApp
from urllib import urlencode
from urllib2 import urlopen, Request

class Repository(object):
    URI_HAS_MODEL = 'info:fedora/fedora-system:def/model#hasModel'

    def __init__(self, root, username=None, password=<PASSWORD>):
        self.fedora_root = root
        self.username = username
        self.password = password

    @property
    def risearch(self):
        return ResourceIndex(self.fedora_root, self.username, self.password)

    def get_objects_with_cmodel(self, cmodel_uri, type=None):
        uris = self.risearch.get_subjects(self.URI_HAS_MODEL, cmodel_uri)
        return [ self.get_object(uri, type) for uri in uris ]

    def get_object(self, pid, type=None):
        if type is None:
            type = DigitalObject
        if pid.startswith('info:fedora/'): # passed a uri
            pid = pid[len('info:fedora/'):]
        return type(pid, self.fedora_root, self.username, self.password)


def read_uri(uri):
    return urlopen(uri).read()

def auth_headers(username, password):
    if username and password:
        token = b64encode('%s:%s' % (username, password))
        return { 'Authorization': 'Basic ' + token }
    else:
        return {}

def make_reader_with_auth(username, password):
    def read_uri_with_auth(uri):
        request = Request(uri, headers=auth_headers(username, password))
        return urlopen(request).read()
    return read_uri_with_auth

def make_rdf_reader_with_auth(username, password):
    def read_rdf_uri_with_auth(uri):
        parser = RDF.Parser('rdfxml')
        request = Request(uri, headers=auth_headers(username, password))
        data = urlopen(request).read()
        return parser.parse_string_as_stream(data, uri)
    return read_rdf_uri_with_auth

class DigitalObject(object):
    def __init__(self, pid, fedora_root, username=None, password=<PASSWORD>):
        self.fedora_root = fedora_root
        self.pid = pid
        self.username = username
        self.password = password

    def __str__(self):
        return self.pid

    def __repr__(self):
        return '<%s %s>' % (self.__class__.__name__, self.pid)

    def http_auth(self):
        if self.username and self.password:
            return 'Basic ' + b64encode('%s:%s' % (self.username, self.password))

    @property
    def api_a_lite(self):
        return API_A_LITE(self.fedora_root, self.username, self.password)

    @property
    def api_m(self):
        return make_service_client(self.fedora_root + 'services/management', API_M())

    @property
    def rest_api(self):
        return REST_API(self.fedora_root, self.username, self.password)

    def get_datastream(self, ds_name, read=None):
        return self.api_a_lite.getDatastreamDissemination(self.pid, ds_name, read)

    def get_datastreams_as_xml(self):
        return self.rest_api.listDatastreams(self.pid)

    def get_relationships(self):
        read = make_rdf_reader_with_auth(self.username, self.password)
        return self.get_datastream('RELS-EXT', read)

    def add_relationship(self, rel_uri, object):
        obj_is_literal = True
        if isinstance(object, DigitalObject):
            object = object.pid
            obj_is_literal = False
        elif isinstance(object, str) and object.startswith('info:fedora/'):
            object = object[len('info:fedora/'):]
            obj_is_literal = False

        extra_headers = auth_headers(self.username, self.password)
        return self.api_m.addRelationship(self.pid, rel_uri, object, obj_is_literal, **extra_headers)


class HTTP_API_Base(object):
    def __init__(self, root, username=None, password=None):
        self.fedora_root = root
        self.read_uri = make_reader_with_auth(username, password)

    def read_relative_uri(self, relative_uri, read=None):
        read = read or self.read_uri
        return read(self.fedora_root + relative_uri)
        

class API_A_LITE(HTTP_API_Base):
    def getDatastreamDissemination(self, pid, ds_name, read=None):
        return self.read_relative_uri('get/%s/%s' % (pid, ds_name), read)


class REST_API(HTTP_API_Base):
    def listDatastreams(self, pid):
        return self.read_relative_uri('objects/%s/datastreams.xml' % (pid,))


class API_M(SimpleWSGISoapApp):
    # FIXME: also accepts an optional String datatype
    @soapmethod(
            soap_types.String,  # pid
            soap_types.String,  # relationship
            soap_types.String,  # object
            soap_types.Boolean, # isLiteral
            _returns = soap_types.Boolean)
    def addRelationship(self, pid, relationship, object, isLiteral):
        pass


class ResourceIndex(object):
    def __init__(self, root, username=None, password=<PASSWORD>):
        self.fedora_root = root
        self.username = username
        self.password = password

    def find_statements(self, spo_query):
        risearch_uri = self.fedora_root + '/risearch?'
        http_args = {
            'type': 'triples',
            'lang': 'spo',
            'format': 'N-Triples',<commit_msg>fedora updates:  * use xml objects for datastreams  * wrap and compose readers a little more cleanly  * add findObjects binding with DigitalObject wrapping<commit_after>import RDF
from base64 import standard_b64encode as b64encode
from eulcore import xmlmap
from soaplib.client import make_service_client
from soaplib.serializers import primitive as soap_types
from soaplib.service import soapmethod
from soaplib.wsgi_soap import SimpleWSGISoapApp
from urllib import urlencode
from urllib2 import urlopen, Request

# a repository object, basically a handy facade for easy api access

class Repository(object):
    URI_HAS_MODEL = 'info:fedora/fedora-system:def/model#hasModel'

    def __init__(self, root, username=None, password=<PASSWORD>):
        self.fedora_root = root
        self.username = username
        self.password = password

    @property
    def risearch(self):
        return ResourceIndex(self.fedora_root, self.username, self.password)

    @property
    def rest_api(self):
        return REST_API(self.fedora_root, self.username, self.password)

    def get_objects_with_cmodel(self, cmodel_uri, type=None):
        uris = self.risearch.get_subjects(self.URI_HAS_MODEL, cmodel_uri)
        return [ self.get_object(uri, type) for uri in uris ]

    def get_object(self, pid, type=None):
        if type is None:
            type = DigitalObject
        if pid.startswith('info:fedora/'): # passed a uri
            pid = pid[len('info:fedora/'):]
        return type(pid, self.fedora_root, self.username, self.password)

    def find_objects(self, type=None, **kwargs):
        type = type or DigitalObject

        # FIXME: query production here is frankly sketchy
        query = ' '.join([ '%s~%s' % (k, v) for k, v in kwargs.iteritems() ])
        read = parse_xml_obj(add_auth(read_uri, self.username, self.password),
                             SearchResults)

        chunk = self.rest_api.findObjects(query, read=read)
        while True:
            for result in chunk.results:
                yield type(result.pid, self.fedora_root, self.username, self.password)

            if chunk.session_token:
                chunk = self.rest_api.findObjects(query, session_token=chunk.session_token, read=read)
            else:
                break

# xml objects to wrap around xml returns from fedora

class ObjectDatastream(xmlmap.XmlObject):
    dsid = xmlmap.XPathString('@dsid')
    label = xmlmap.XPathString('@label')
    mimeType = xmlmap.XPathString('@mimeType')

class ObjectDatastreams(xmlmap.XmlObject):
    pid = xmlmap.XPathString('@pid')
    datastreams = xmlmap.XPathNodeList('datastream', ObjectDatastream)

class SearchResult(xmlmap.XmlObject):
    def __init__(self, dom_node, context=None):
        if context is None:
            context = xmlmap.Context(dom_node, processorNss={'res': 'http://www.fedora.info/definitions/1/0/types/'})
        xmlmap.XmlObject.__init__(self, dom_node, context)

    pid = xmlmap.XPathString('res:pid')

class SearchResults(xmlmap.XmlObject):
    def __init__(self, dom_node, context=None):
        if context is None:
            context = xmlmap.Context(dom_node, processorNss={'res': 'http://www.fedora.info/definitions/1/0/types/'})
        xmlmap.XmlObject.__init__(self, dom_node, context)

    session_token = xmlmap.XPathString('res:listSession/res:token')
    cursor = xmlmap.XPathInteger('res:listSession/res:cursor')
    expiration_date = xmlmap.XPathDate('res:listSession/res:expirationDate')
    results = xmlmap.XPathNodeList('res:resultList/res:objectFields', SearchResult)

# readers used internally to affect how we interpret network data from fedora

def read_uri(uri):
    return urlopen(uri).read()

def auth_headers(username, password):
    if username and password:
        token = b64encode('%s:%s' % (username, password))
        return { 'Authorization': 'Basic ' + token }
    else:
        return {}

def add_auth(reader, username, password):
    def read_uri_with_auth(uri):
        request = Request(uri, headers=auth_headers(username, password))
        return reader(request)
    return read_uri_with_auth

def parse_rdf(reader):
    def read_rdf_uri(uri):
        parser = RDF.Parser('rdfxml')
        data = reader(uri)
        return parser.parse_string_as_stream(data, uri)
    return read_rdf_uri

def parse_xml_obj(reader, xml_class):
    def read_xml_uri(uri):
        data = reader(uri)
        doc = xmlmap.parseString(data, uri)
        return xml_class(doc.documentElement)
    return read_xml_uri

# a single digital object in a repo; basically another facade for api access

class DigitalObject(object):
    def __init__(self, pid, fedora_root, username=None, password=<PASSWORD>):
        self.fedora_root = fedora_root
        self.pid = pid
        self.username = username
        self.password = password

    def __str__(self):
        return self.pid

    def __repr__(self):
        return '<%s %s>' % (self.__class__.__name__, self.pid)

    @property
    def api_a_lite(self):
        return API_A_LITE(self.fedora_root, self.username, self.password)

    @property
    def api_m(self):
        return make_service_client(self.fedora_root + 'services/management', API_M())

    @property
    def rest_api(self):
        return REST_API(self.fedora_root, self.username, self.password)

    def get_datastream(self, ds_name, read=None):
        return self.api_a_lite.getDatastreamDissemination(self.pid, ds_name, read)

    def get_datastreams(self):
        read = parse_xml_obj(add_auth(read_uri, self.username, self.password),
                             ObjectDatastreams)
        dsobj = self.rest_api.listDatastreams(self.pid, read)
        return dict([ (ds.dsid, ds) for ds in dsobj.datastreams ])

    def get_relationships(self):
        read = parse_rdf(add_auth(read_uri, self.username, self.password))
        return self.get_datastream('RELS-EXT', read)

    def add_relationship(self, rel_uri, object):
        obj_is_literal = True
        if isinstance(object, DigitalObject):
            object = object.pid
            obj_is_literal = False
        elif isinstance(object, str) and object.startswith('info:fedora/'):
            object = object[len('info:fedora/'):]
            obj_is_literal = False

        extra_headers = auth_headers(self.username, self.password)
        return self.api_m.addRelationship(self.pid, rel_uri, object, obj_is_literal, **extra_headers)

# fedora apis

class HTTP_API_Base(object):
    def __init__(self, root, username=None, password=<PASSWORD>):
        self.fedora_root = root
        self.read_uri = add_auth(read_uri, username, password)

    def read_relative_uri(self, relative_uri, read=None):
        read = read or self.read_uri
        return read(self.fedora_root + relative_uri)
        

class API_A_LITE(HTTP_API_Base):
    def getDatastreamDissemination(self, pid, ds_name, read=None):
        return self.read_relative_uri('get/%s/%s' % (pid, ds_name), read)


class REST_API(HTTP_API_Base):
    def findObjects(self, query, pid=True, session_token=None, read=None):
        http_args = {
            'query': query,
            'resultFormat': 'xml',
        }
        if pid:
            http_args['pid'] = 'true'
        if session_token:
            http_args['sessionToken'] = session_token
        return self.read_relative_uri('objects?' + urlencode(http_args), read)
        
    def listDatastreams(self, pid, read=None):
        return self.read_relative_uri('objects/%s/datastreams.xml' % (pid,), read)


class API_M(SimpleWSGISoapApp):
    # FIXME: also accepts an optional String datatype
    @soapmethod(
            soap_types.String,  # pid
            soap_types.String,  # relationship
            soap_types.String,  # object
            soap_types.Boolean, # isLiteral
            _returns = soap_types.Boolean)
    def addRelationship(self, pid, relationship, object, isLiteral):
        pass


class ResourceIndex(object):
    def __init__(self, root, username=None, password=<PASSWORD>):
        self.fedora_root = root
        self.username = username
        self.password = password

    def find_statements(self, spo_query):
        risearch_uri = self.fedora_root + '/risearch?'
        http_args = {
            'type': 'triples',
            'lang': 'spo',
            'format': 'N-Triples',
{'fraction_non_alphanumeric': 0.10178715893211737, 'fraction_numerical': 0.0017650952415974113, 'mean_word_length': 3.1356447688564475, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 36}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3788988', 'n_tokens_mistral': 4248, 'n_tokens_neox': 4170, 'n_words': 931}