<commit_before>'''
Copyright 2011 SRI International

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Created on Mar 10, 2011

@author: jklo
'''
import logging, iso8601
import couchdb
from lr.lib.base import render
from lr.lib.harvest import harvest
from lr.lib.oaipmherrors import IdDoesNotExistError, NoMetadataFormats
import lr.lib.helpers as h
from lr.model.base_model import appConfig

log = logging.getLogger(__name__)

class oaipmh(harvest):
    '''
    Utility class to provide OAI-PMH results from Learning Registy
    '''
    def __init__(self, server=appConfig['couchdb.url'], database="resource_data"):
        '''
        Constructor
        '''
        self.server = couchdb.Server(server)
        self.db = self.server[database]
    
    
    
    def list_records(self,metadataPrefix,from_date=None, until_date=None):
        opts = {};

        if from_date != None:
            from_date = from_date.replace(tzinfo=None)
            opts["startkey"] = [metadataPrefix, from_date.isoformat()]
        else:
            # empty string should sort before anything else.
            opts["startkey"] = [metadataPrefix, None]
        
        if until_date != None:
            until_date = until_date.replace(tzinfo=None)
            opts["endkey"] = [metadataPrefix, until_date.isoformat()]
        else:
            # somewhat of an hack... since I don't know the timestamp, and couch stores things
            # in alphabetical order, a string sequence with all capital Z's should always sort last.
            opts["endkey"] = [metadataPrefix, {}];
        
        
        view_data = self.db.view('oai-pmh/list-records', **opts)
        return map(lambda row: row["value"], view_data)
    
    def list_identifiers(self,metadataPrefix,from_date=None, until_date=None ):
        opts = {};
        import logging
        log = logging.getLogger(__name__)

        if from_date != None:
            opts["startkey"] = [metadataPrefix, from_date.isoformat()]
        else:
            # empty string should sort before anything else.
            opts["startkey"] = [metadataPrefix, None]
        
        if until_date != None:
            opts["endkey"] = [metadataPrefix, until_date.isoformat()]
        else:
            # somewhat of an hack... since I don't know the timestamp, and couch stores things
            # in alphabetical order, a string sequence with all capital Z's should always sort last.
            opts["endkey"] = [metadataPrefix, {}];
        
        log.info("opts: "+ repr(opts))
        view_data = self.db.view('oai-pmh/list-identifiers', **opts)
        return map(lambda row: { "doc_ID": row["id"], "node_timestamp": row["key"][1] }, view_data)
    
    def list_metadata_formats(self, identity=None, by_doc_ID=False, verb="ListMetadataFormats"):
        try:
            opts = {}
            if identity != None:
                if by_doc_ID == True: 
                    byID = "by_doc_ID" 
                else: 
                    byID = "by_resource_ID"
                opts["key"] = [byID, identity]
                
                view_data = self.db.view('oai-pmh/get-records', **opts)
                if len(view_data) == 0:
                    raise IdDoesNotExistError(verb)
                formats = [];
                for res in view_data:
                    for schema in res.value["payload_schema"]:
                        formats.append({"metadataPrefix":schema, "schemas":[res.value["payload_schema_locator"]]})
                return formats
            
            else:
                opts["group"] = True
                opts["group_level"] = 1
                view_data = self.db.view('oai-pmh/list-metadata-formats', **opts)
                return map(lambda doc: {"metadataPrefix": doc.key[0], "schemas": doc.value}, view_data)
                
        except Exception as e:
            raise e
        
    def identify(self, database="node"):
        ret = None
        ident = {}
        try:
            db2 = self.server[database]
            description = db2["node_description"]
            
            ident["repositoryName"] = description["node_description"]
            ident["adminEmail"] = description["node_admin_identity"]
            # TODO: This should map to the deleted_data_policy from the node_policy from the
            #       network node description
            ident["deletedRecord"] = "transient"
            ident["granularity"] = h.getDatetimePrecision()
            opts = {
                    "group": True,
                    "limit": 1
                    }
            
            view_data = self.db.view('oai-pmh/identify-timestamp', **opts)
            if len(view_data) > 0:
                ident["earliestDatestamp"] = iso8601.parse_date(list(view_data)[0].key)
            else:
                ident["earliestDatestamp"] = datetime.utcnow()
            
            ret = ident
        except Exception as e:
            log.exception("Problem determining OAI Identity")
        <commit_msg>fixed an import problem found on an edge condition (no data) in oai-pmh lib.<commit_after>'''
Copyright 2011 SRI International

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Created on Mar 10, 2011

@author: jklo
'''
import logging, iso8601
import couchdb
from lr.lib.base import render
from lr.lib.harvest import harvest
from lr.lib.oaipmherrors import IdDoesNotExistError, NoMetadataFormats
import lr.lib.helpers as h
from lr.model.base_model import appConfig

log = logging.getLogger(__name__)

class oaipmh(harvest):
    '''
    Utility class to provide OAI-PMH results from Learning Registy
    '''
    def __init__(self, server=appConfig['couchdb.url'], database="resource_data"):
        '''
        Constructor
        '''
        self.server = couchdb.Server(server)
        self.db = self.server[database]
    
    
    
    def list_records(self,metadataPrefix,from_date=None, until_date=None):
        opts = {};

        if from_date != None:
            from_date = from_date.replace(tzinfo=None)
            opts["startkey"] = [metadataPrefix, from_date.isoformat()]
        else:
            # empty string should sort before anything else.
            opts["startkey"] = [metadataPrefix, None]
        
        if until_date != None:
            until_date = until_date.replace(tzinfo=None)
            opts["endkey"] = [metadataPrefix, until_date.isoformat()]
        else:
            # somewhat of an hack... since I don't know the timestamp, and couch stores things
            # in alphabetical order, a string sequence with all capital Z's should always sort last.
            opts["endkey"] = [metadataPrefix, {}];
        
        
        view_data = self.db.view('oai-pmh/list-records', **opts)
        return map(lambda row: row["value"], view_data)
    
    def list_identifiers(self,metadataPrefix,from_date=None, until_date=None ):
        opts = {};
        import logging
        log = logging.getLogger(__name__)

        if from_date != None:
            opts["startkey"] = [metadataPrefix, from_date.isoformat()]
        else:
            # empty string should sort before anything else.
            opts["startkey"] = [metadataPrefix, None]
        
        if until_date != None:
            opts["endkey"] = [metadataPrefix, until_date.isoformat()]
        else:
            # somewhat of an hack... since I don't know the timestamp, and couch stores things
            # in alphabetical order, a string sequence with all capital Z's should always sort last.
            opts["endkey"] = [metadataPrefix, {}];
        
        log.info("opts: "+ repr(opts))
        view_data = self.db.view('oai-pmh/list-identifiers', **opts)
        return map(lambda row: { "doc_ID": row["id"], "node_timestamp": row["key"][1] }, view_data)
    
    def list_metadata_formats(self, identity=None, by_doc_ID=False, verb="ListMetadataFormats"):
        try:
            opts = {}
            if identity != None:
                if by_doc_ID == True: 
                    byID = "by_doc_ID" 
                else: 
                    byID = "by_resource_ID"
                opts["key"] = [byID, identity]
                
                view_data = self.db.view('oai-pmh/get-records', **opts)
                if len(view_data) == 0:
                    raise IdDoesNotExistError(verb)
                formats = [];
                for res in view_data:
                    for schema in res.value["payload_schema"]:
                        formats.append({"metadataPrefix":schema, "schemas":[res.value["payload_schema_locator"]]})
                return formats
            
            else:
                opts["group"] = True
                opts["group_level"] = 1
                view_data = self.db.view('oai-pmh/list-metadata-formats', **opts)
                return map(lambda doc: {"metadataPrefix": doc.key[0], "schemas": doc.value}, view_data)
                
        except Exception as e:
            raise e
        
    def identify(self, database="node"):
        ret = None
        ident = {}
        try:
            db2 = self.server[database]
            description = db2["node_description"]
            
            ident["repositoryName"] = description["node_description"]
            ident["adminEmail"] = description["node_admin_identity"]
            # TODO: This should map to the deleted_data_policy from the node_policy from the
            #       network node description
            ident["deletedRecord"] = "transient"
            ident["granularity"] = h.getDatetimePrecision()
            opts = {
                    "group": True,
                    "limit": 1
                    }
            
            view_data = self.db.view('oai-pmh/identify-timestamp', **opts)
            if len(view_data) > 0:
                ident["earliestDatestamp"] = iso8601.parse_date(list(view_data)[0].key)
            else:
                from datetime import datetime
                ident["earliestDatestamp"] = datetime.utcnow()
            
            ret = ident
        except Exception as e:
            log.exception("Problem determining OAI Identity")
        
{'fraction_non_alphanumeric': 0.10128863656844192, 'fraction_numerical': 0.005587095611426511, 'mean_word_length': 2.03224043715847, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3904838', 'n_tokens_mistral': 3061, 'n_tokens_neox': 2943, 'n_words': 831}