<commit_before>## $Id$

## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""OAI interface for CDS Invenio written in Python compliant with OAI-PMH2.0"""

__revision__ = "$Id$"

import cPickle
import string
from string import split
import os
import re
import urllib
import sys
import time
import md5

from invenio.config import \
     CFG_OAI_DELETED_POLICY, \
     CFG_OAI_EXPIRE, \
     CFG_OAI_IDENTIFY_DESCRIPTION, \
     CFG_OAI_ID_FIELD, \
     CFG_OAI_ID_PREFIX, \
     CFG_OAI_LOAD, \
     CFG_OAI_SAMPLE_IDENTIFIER, \
     CFG_OAI_SET_FIELD, \
     adminemail, \
     cachedir, \
     cdsname, \
     supportemail, \
     version, \
     weburl
from invenio.oai_repository_config import *
from invenio.dbquery import run_sql
from invenio.search_engine import record_exists

verbs = {
    "Identify"            : [""],
    "ListSets"            : ["resumptionToken"],
    "ListMetadataFormats" : ["resumptionToken"],
    "ListRecords"         : ["resumptionToken"],
    "ListIdentifiers"     : ["resumptionToken"],
    "GetRecord"	          : [""]
}

params = {
    "verb" : ["Identify","ListIdentifiers","ListSets","ListMetadataFormats","ListRecords","GetRecord"],
    "metadataPrefix" : ["","oai_dc","marcxml"],
    "from" :[""],
    "until":[""],
    "set" :[""],
    "identifier": [""]
}

def encode_for_xml(strxml):
    "Encode special chars in string for XML-compliancy."

    if strxml is None:
        return strxml
    else:
        strxml = string.replace(strxml, '&', '&amp;')
        strxml = string.replace(strxml, '<', '&lt;')
        return strxml

def escape_space(strxml):
    "Encode special chars in string for URL-compliancy."

    strxml = string.replace(strxml, ' ', '%20')
    return strxml

def encode_for_url(strxml):
    "Encode special chars in string for URL-compliancy."

    strxml = string.replace(strxml, '%', '%25')
    strxml = string.replace(strxml, ' ', '%20')
    strxml = string.replace(strxml, '?', '%3F')
    strxml = string.replace(strxml, '#', '%23')
    strxml = string.replace(strxml, '=', '%3D')
    strxml = string.replace(strxml, '&', '%26')
    strxml = string.replace(strxml, '/', '%2F')
    strxml = string.replace(strxml, ':', '%3A')
    strxml = string.replace(strxml, ';', '%3B')
    strxml = string.replace(strxml, '+', '%2B')

    return strxml

def oai_header(args, verb):
    "Print OAI header"

    out = ""

    out = out + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "\n"
    out = out + "<OAI-PMH xmlns=\"http://www.openarchives.org/OAI/2.0/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd\">\n"

    out = out + " <responseDate>" + oaigetresponsedate() + "</responseDate>\n"

    if verb:
        out = out + " <request verb=\"%s\">%s</request>\n" % (verb, oaigetrequesturl(args))
        out = out + " <%s>\n" % verb
    else:
        out = out + " <request>%s</request>\n" % (oaigetrequesturl(args))

    return out

def oai_footer(verb):
    "Print OAI footer"

    out = ""

    if verb:
        out = "%s </%s>\n" % (out, verb)
    out = out + "</OAI-PMH>\n"

    return out

def oai_error_header(args, verb):
    "Print OAI header"

    out = ""

###    out = "Content-Type: text/xml\n\n"
    out = out + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "\n"
    out = out + "<OAI-PMH xmlns=\"http://www.openarchives.org/OAI/2.0/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd\">\n"

    out = out + " <responseDate>" + oaigetresponsedate() + "</responseDate>\n"
    out = out + " <request verb=\"%s\">%s</request>\n" % (verb, oaigetrequesturl(args))

    return out

def oai_error_footer(verb):
    "Print OAI footer"

    out  = verb
    out  = "</OAI-PMH>\n"
    return out

def get_field(sysno, field):
    "Gets list of field 'field' for the record with 'sysno' system number."

    out   = []
    digit = field[0:2]

    bibbx = "bib%sx" % digit
    bibx  = "bibrec_bib%sx" % digit
    query = "SELECT bx.value FROM %s AS bx, %s AS bibx WHERE bibx.id_bibrec='%s' AND bx.id=bibx.id_bibxxx AND bx.tag='%s'" % (bibbx, bibx, sysno, field)

    res = run_sql(query)

    for row in res:

        out.append(row[0])

    return out

def utc_to_localtime(date):
    "Convert UTC to localtime"

    ldate = date.split("T")[0]
    ltime = date.split("T")[1]

    lhour   = ltime.split(":")[0]
    lminute = ltime.split(":")[1]
    lsec    = ltime.split(":")[2]

    lyear   = ldate.split("-")[0]
    lmonth  = ldate.split("-")[1]
    lday    = ldate.split("-")[2]
    
    timetoconvert = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.mktime((string.atoi(lyear), string.atoi(lmonth), string.atoi(lday), string.atoi(lhour), string.atoi(lminute), string.atoi(lsec[:-1]), 0, 0, -1)) - time.timezone + (time.daylight)*3600))

    return timetoconvert

def localtime_to_utc(date):
    "Convert localtime to UTC"

    ldate = date.split(" ")[0]
    ltime = date.split(" ")[1]

    lhour   = ltime.split(":")[0]
    lminute = ltime.split(":")[1]
    lsec    = ltime.split(":")[2]

    lyear   = ldate.split("-")[0]
    lmonth  = ldate.split("-")[1]
    lday    = ldate.split("-")[2]

    timetoconvert = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.mktime((string.atoi(lyear), string.atoi(lmonth), string.atoi(lday), string.atoi(lhour), string.atoi(lminute), string.atoi(lsec), 0, 0, -1))))

    return timetoconvert

def get_creation_date(sysno):
    "Returns the creation date of the record 'sysno'."
    out   = ""
    res = run_sql("SELECT DATE_FORMAT(creation_date, '%%Y-%%m-%%d %%H:%%i:%%s') FROM bibrec WHERE id=%s", (sysno,), 1)
    if res[0][0]:
        out = localtime_to_utc(res[0][0])
    return out

def get_modification_date(sysno):
    "Returns the date of last modification for the record 'sysno'."
    out = ""
    res = run_sql("SELECT DATE_FORMAT(modification_date,'%%Y-%%m-%%d %%H:%%i:%%s') FROM bibrec WHERE id=%s", (sysno,), 1)
    if res and res[0][0]:
        out = localtime_to_utc(res[0][0])
    return out

def get_earliest_datestamp():
    "Get earliest datestamp in the database"
    out = ""
    res = run_sql("SELECT MIN(DATE_FORMAT(creation_date,'%%Y-%%m-%%d %%H:%%i:%%s')) FROM bibrec", (), 1)
    if res[0][0]:
        out = localtime_to_utc(res[0][0])
    return out

def check_date(date, dtime="T00:00:00Z"):
    "Check if the date has a correct format"

    if(re.sub("[0123456789\-:TZ]", "", date) == ""):
        if len(date) == 10:
            date = date + dtime
        if len(date) == 20:
            date = utc_to_localtime(date)
        else:
            date = ""
    else:
        date = ""
    
    return date

def print_record(sysno, format='marcxml'):
    """Prints record 'sysno' formatted according to 'format'.

    - if record does not exist, return nothing.

    - if record has been deleted and CFG_OAI_DELETED_POLICY is
      'transient' or 'deleted', then return only header, with status
      'deleted'.
      
    - if record has been deleted and CFG_OAI_DELETED_POLICY is 'no',
      then return nothing.
    """
    
    out = ""
    
    # sanity check:
    if not record_exists(sysno):
        return

    if (format == "dc") or (format == "oai_dc"):
        format = "xd"

    # print record opening tags:
    
    out = out + "  <record>\n"

    if record_exists(sysno) == -1: # Deleted?
        if CFG_OAI_DELETED_POLICY == "persistent" or \
               CFG_OAI_DELETED_POLICY == "transient":
            out = out + "    <header status=\"deleted\">\n"
        else:
            return
    else:
        out = out + "   <header>\n"

    for ident in get_field(sysno, CFG_OAI_ID_FIELD):
        out = "%s    <identifier>%s</identifier>\n" % (out, escape_space(ident))
    out = "%s    <datestamp>%s</datestamp>\n" % (out, get_modification_date(sysno))
    for set in get_field(sysno, CFG_OAI_SET_FIELD):
        out = "%s    <setSpec>%s</setSpec>\n" % (out, set)
    out = out + "   </header>\n"

    if record_exists(sysno) == -1: # Deleted?
        pass
    else:
        out = out + "   <metadata>\n"

        if format == "marcxml":
            out = out + "    <marc:record xmlns:marc=\"http://www.loc.gov/MARC21/slim\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd\" type=\"Bibliographic\">"
            out = out + "     <marc:leader>00000coc  2200000uu 4500</marc:leader>"
            ## MARC21 and XML formats, possibley OAI -- they are not in "bibfmt" table; so fetch all the data from "bibXXx" tables:

            if format == "marcxml":

                out = "%s     <marc:controlfield tag=\"001\">%d</marc:controlfield>\n" % (out, int(sysno))

            for digit1 in range(0, 10):
                for digit2 in range(0, 10):
                    bibbx = "bib%d%dx" % (digit1, digit2)
                    bibx = "bibrec_bib%d%dx" % (digit1, digit2)
                    query = "SELECT b.tag,b.value,bb.field_number FROM %s AS b, %s AS bb "\
                            "WHERE bb.id_bibrec='%s' AND b.id=bb.id_bibxxx AND b.tag LIKE '%s%%' "\
                            "ORDER BY bb.field_number, b.tag ASC" % (bibbx, bibx, sysno, str(digit1)+str(digit2))
                    res = run_sql(query)
                    field_number_old = -999
                    field_old = ""
                    for row in res:
                        field, value, field_number = row[0], row[1], row[2]
                        ind1, ind2 = field[3], field[4]
                        if ind1 == "_":
                            ind1 = " "
                        if ind2 == "_":
                            ind2 = " "                        
                        # print field tag
                        if field_number != field_number_old or field[:-1] != field_old[:-1]:
                            if format == "marcxml":

                                if field_number_old != -999:
                                    if field_old[0:2] == "00":
                                        out = out + "     </marc:controlfield>\n"
                                    else:
                                        out = out + "     </marc:datafield>\n"

                                if field[0:2] == "00":
                                    out = "%s     <marc:controlfield tag=\"%s\">\n" % (out, encode_for_xml(field[0:3]))
                                else:
                                    out = "%s     <marc:datafield tag=\"%s\" ind1=\"%s\" ind2=\"%s\">\n" % (out, encode_for_xml(field[0:3]), encode_for_xml(ind1).lower(), encode_for_xml(ind2).lower())


                            field_number_old = field_number
                            field_old = field
                        # print subfield value
                        if format == "marcxml":
                            value = encode_for_xml(value)

                            if(field[0:2] == "00"):
                                out = "%s      %s\n" % (out, value)
                            else:
                                out = "%s      <marc:subfield code=\"%s\">%s</marc:subfield>\n" % (out, encode_for_xml(field[-1:]), value)

                               
                        # fetch next subfield
                    # all fields/subfields printed in this run, so close the tag:
                    if (format == "marcxml") and field_number_old != -999:
                        if field_old[0:2] == "00":
                            out = out + "     </marc:controlfield>\n"
                        else:
                            out = out + "     </marc:datafield>\n"
                        
            out = out + "    </marc:record>\n"

        elif format == "xd":
        # XML Dublin Core format, possibly OAI -- select only some bibXXx fields:
            out = out + "       <oaidc:dc xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:oaidc=\"http://www.openarchives.org/OAI/2.0/oai_dc/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\">\n"

            for field_ in get_field(sysno, "041__a"):
                out =  "%s         <dc:language>%s</dc:language>\n" % (out, field_)

            for field_ in get_field(sysno, "100__a"):
                out =  "%s         <dc:creator>%s</dc:creator>\n" % (out, encode_for_xml(field_))
 
            for field_ in get_field(sysno, "700__a"):
                out =  "%s         <dc:creator>%s</dc:creator>\n" % (out, encode_for_xml(field_))

            for field_ in get_field(sysno, "245__a"):
                out =  "%s         <dc:title>%s</dc:title>\n" % (out, encode_for_xml(field_))

            for field_ in get_field(sysno, "111__a"):
                out =  "%s         <dc:title>%s</dc:title>\n" % (out, encode_for_xml(field_))

            for field_ in get_field(sysno, "65017a"):
                out =  "%s         <dc:subject>%s</dc:subject>\n" % (out, encode_for_xml(field_))

            for field_ in get_field(sysno, "8564_u"):
                out =  "%s         <dc:identifier>%s</dc:identifier>\n" % (out, encode_for_xml(escape_space(field_)))
        
            for field_ in get_field(sysno, "520__a"):
                out = "%s         <dc:description>%s</dc:description>\n" % (out, encode_for_xml(field_))

            date = get_creation_date(sysno)
 
            out = "%s         <dc:date>%s</dc:date>\n" % (out, date)
            out = out + "    </oaidc:dc>\n"

    # print record closing tags:
    
        out = out + "   </metadata>\n"

    out = out + "  </record>\n"

    return out

def oailistmetadataformats(args):
    "Generates response to oailistmetadataformats verb."

    arg = parse_args(args)

    out = ""

    flag = 1 # list or not depending on identifier

    if arg['identifier'] != "":

        flag = 0

        sysno = oaigetsysno(arg['identifier'])

        if record_exists(sysno) == 1 or \
               (record_exists(sysno) == -1 and CFG_OAI_DELETED_POLICY != "no"):

            flag = 1

        else:

            out = out + oai_error("idDoesNotExist","invalid record Identifier")
            out = oai_error_header(args, "ListMetadataFormats") + out + oai_error_footer("ListMetadataFormats")
            return out

    if flag:
        out = out + "   <metadataFormat>\n"
        out = out + "    <metadataPrefix>oai_dc</metadataPrefix>\n"
        out = out + "    <schema>http://www.openarchives.org/OAI/1.1/dc.xsd</schema>\n"
        out = out + "    <metadataNamespace>http://purl.org/dc/elements/1.1/</metadataNamespace>\n"
        out = out + "   </metadataFormat>\n"
        out = out + "   <metadataFormat>\n"
        out = out + "    <metadataPrefix>marcxml</metadataPrefix>\n"
        out = out + "    <schema>http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd</schema>\n"
        out = out + "    <metadataNamespace>http://www.loc.gov/MARC21/slim</metadataNamespace>\n"
        out = out + "   </metadataFormat>\n"

    out = oai_header(args, "ListMetadataFormats") + out + oai_footer("ListMetadataFormats")
    return out


def oailistrecords(args):
    "Generates response to oailistrecords verb."

    arg = parse_args(args)

    out = ""

    sysnos = []
    sysno  = []

    # check if the resumptionToken did not expire
    if arg['resumptionToken']:
        filename = "%s/RTdata/%s" % (cachedir, arg['resumptionToken'])
        if os.path.exists(filename) == 0:
            out = oai_error("badResumptionToken", "ResumptionToken expired")
            out = oai_error_header(args, "ListRecords") + out + oai_error_footer("ListRecords")
            return out

    if arg['resumptionToken'] != "":
        sysnos = oaicacheout(arg['resumptionToken'])
        arg['metadataPrefix'] = sysnos.pop()
    else:
        sysnos = oaigetsysnolist(arg['set'], arg['from'], arg['until'])

    if len(sysnos) == 0: # noRecordsMatch error
        
        out = out + oai_error("noRecordsMatch", "no records correspond to the request")
        out = oai_error_header(args, "ListRecords") + out + oai_error_footer("ListRecords")
        return out

    i = 0
    for sysno_ in sysnos:
        if sysno_:
            if not (record_exists(sysno_) == -1 and CFG_OAI_DELETED_POLICY == "no"): 
                i = i + 1 # Increment limit only if record is returned
           
            if i > CFG_OAI_LOAD:          # cache or write?
                if i == CFG_OAI_LOAD + 1: # resumptionToken?
                    arg['resumptionToken'] = oaigenresumptionToken()
                    extdate = oaigetresponsedate(CFG_OAI_EXPIRE)
                    if extdate:
                        out = "%s <resumptionToken expirationDate=\"%s\">%s</resumptionToken>\n" % (out, extdate, arg['resumptionToken'])
                    else:
                        out = "%s <resumptionToken>%s</resumptionToken>\n" % (out, arg['resumptionToken'])
                sysno.append(sysno_)
            else:
                res = print_record(sysno_, arg['metadataPrefix'])
                if res:
                    out += res

    if i > CFG_OAI_LOAD:
        oaicacheclean()
        sysno.append(arg['metadataPrefix'])
        oaicachein(arg['resumptionToken'], sysno)

    out = oai_header(args, "ListRecords") + out + oai_footer("ListRecords")
    return out

def oailistsets(args):
    "Lists available sets for OAI metadata harvesting."

    out = ""

    # note: no flow control in ListSets

    sets = get_sets()

    for set_ in sets:

        out = out + "  <set>\n"
        out = "%s    <setSpec>%s</setSpec>\n" % (out, set_[0])
        out = "%s    <setName>%s</setName>\n" % (out, set_[1])
        if set_[2]:
            out = "%s    <setDescription>%s</setDescription>\n" % (out, set_[2])
        out = out + "   </set>\n"

    out = oai_header(args, "ListSets") + out + oai_footer("ListSets")

    return out

    
def oaigetrecord(args):
    """Returns record 'identifier' according to 'metadataPrefix' format for OAI metadata harvesting.
    
    - if record does not exist, return oai_error 'idDoesNotExist'.

    - if record has been deleted and CFG_OAI_DELETED_POLICY is
      'transient' or 'deleted', then return only header, with status
      'deleted'.
      
    - if record has been deleted and CFG_OAI_DELETED_POLICY is 'no',
      then return oai_error 'idDoesNotExist'.
    """
    
    arg = parse_args(args)
    out = ""
    sysno = oaigetsysno(arg['identifier'])

    if record_exists(sysno) == 1 or \
           (record_exists(sysno) == -1 and CFG_OAI_DELETED_POLICY != 'no'):
        datestamp = get_modification_date(sysno)
        out += print_record(sysno, arg['metadataPrefix'])
    else:
        out = out + oai_error("idDoesNotExist", "invalid record Identifier")
        out = oai_error_header(args, "GetRecord") + out + oai_error_footer("GetRecord")
    return out

    out = oai_header(args, "GetRecord") + out + oai_footer("GetRecord")

    return out


def oailistidentifiers(args):
    "Prints OAI response to the ListIdentifiers verb."

    arg = parse_args(args)

    out = ""

    sysno  = []
    sysnos = []

    if arg['resumptionToken']:
        filename = "%s/RTdata/%s" % (cachedir, arg['resumptionToken'])
        if os.path.exists(filename) == 0:
            out = out + oai_error("badResumptionToken", "ResumptionToken expired")
            out = oai_error_header(args, "ListIdentifiers") + out + oai_error_footer("ListIdentifiers")
            return out

    if arg['resumptionToken']:
        sysnos = oaicacheout(arg['resumptionToken'])
    else:
        sysnos = oaigetsysnolist(arg['set'], arg['from'], arg['until'])

    if len(sysnos) == 0: # noRecordsMatch error
        out = out + oai_error("noRecordsMatch", "no records correspond to the request")
        out = oai_error_header(args, "ListIdentifiers") + out + oai_error_footer("ListIdentifiers")
        return out

    i = 0
    for sysno_ in sysnos:
        if sysno_:
            if not (record_exists(sysno_) == -1 and CFG_OAI_DELETED_POLICY == "no"): 
                i = i + 1 # Increment limit only if record is returned
                
            if i > CFG_OAI_LOAD:           # cache or write?
                if i ==  CFG_OAI_LOAD + 1: # resumptionToken?
                    arg['resumptionToken'] = oaigenresumptionToken()
                    extdate = oaigetresponsedate(CFG_OAI_EXPIRE)
                    if extdate:
                        out = "%s  <resumptionToken expirationDate=\"%s\">%s</resumptionToken>\n" % (out, extdate, arg['resumptionToken'])
                    else:
                        out = "%s  <resumptionToken>%s</resumptionToken>\n" % (out, arg['resumptionToken'])
                sysno.append(sysno_)
            else:
                for ident in get_field(sysno_, CFG_OAI_ID_FIELD):
                    if ident != '':
                        if record_exists(sysno_) == -1: #Deleted?
                            if CFG_OAI_DELETED_POLICY == "persistent" \
                                   or CFG_OAI_DELETED_POLICY == "transient":
                                out = out + "    <header status=\"deleted\">\n"
                        else:
                            out = out + "    <header>\n"
                        out = "%s      <identifier>%s</identifier>\n" % (out, escape_space(ident))
                        out = "%s      <datestamp>%s</datestamp>\n" % (out, get_modification_date(oaigetsysno(ident)))
                        for set in get_field(sysno_, CFG_OAI_SET_FIELD):
                            out = "%s      <setSpec>%s</setSpec>\n" % (out, set)
                        out = out + "    </header>\n"

    if i > CFG_OAI_LOAD:
        oaicacheclean() # clean cache from expired resumptionTokens
        oaicachein(arg['resumptionToken'], sysno)

    out = oai_header(args, "ListIdentifiers") + out + oai_footer("ListIdentifiers")

    return out


def oaiidentify(args):
    "Generates response to oaiidentify verb."
        
    out = ""

    repositoryname        = "  <repositoryName>" + cdsname + "</repositoryName>\n"
    baseurl               = "  <baseURL>%s/oai2d/</baseURL>\n" % weburl
    protocolversion       = "  <protocolVersion>2.0</protocolVersion>\n"
    adminemail            = "  <adminEmail>%s</adminEmail>\n" % supportemail
    earliestdst		  = "  <earliestDatestamp>%s</earliestDatestamp>\n" % get_earliest_datestamp()               
    deletedrecord         = "  <deletedRecord>%s</deletedRecord>\n" % CFG_OAI_DELETED_POLICY
    repositoryidentifier  = "%s" % CFG_OAI_ID_PREFIX
    sampleidentifier      = CFG_OAI_SAMPLE_IDENTIFIER
    identifydescription   = CFG_OAI_IDENTIFY_DESCRIPTION + "\n"

    out = out + repositoryname
    out = out + baseurl
    out = out + protocolversion
    out = out + adminemail
    out = out + earliestdst
    out = out + deletedrecord
    out = out + "  <granularity>YYYY-MM-DDThh:mm:ssZ</granularity>\n"
    #    print "  <compression></compression>\n"
    out = out + CFG_OAI_IDENTIFY_DESCRIPTION

    out = oai_header(args, "Identify") + out + oai_footer("Identify")

    return out

     
def oaigetrequesturl(args):
    "Generates requesturl tag for OAI."

    # re_amp = re.compile('&')

    requesturl = weburl + "/" + "oai2d/"# + "?" + re_amp.sub("&amp;", args)

    return requesturl

def oaigetresponsedate(delay=0):
    "Generates responseDate tag for OAI."
    
    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() + delay))


def oai_error(code, msg):
    "OAI error occured"

    return "<error code=\"%s\">%s</error>\n" % (code, msg)


def oaigetsysno(identifier):
    "Returns the first database BIB ID for the OAI identifier 'identifier', if it exists."
    sysno = None
    if identifier:
        query = "SELECT DISTINCT(bb.id_bibrec) FROM bib%sx AS bx, bibrec_bib%sx AS bb WHERE bx.tag='%s' AND bb.id_bibxxx=bx.id AND bx.value='%s'" % (CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD, identifier)
        res = run_sql(query)
        for row in res:
            sysno = row[0]
    return sysno


def oaigetsysnolist(set, fromdate, untildate):
    "Returns list of system numbers for the OAI set 'set', modified from 'date_from' until 'date_until'."

    out_dict = {} # dict to hold list of out sysnos as its keys

    if set:
        query = "SELECT DISTINCT bibx.id_bibrec FROM bib%sx AS bx LEFT JOIN bibrec_bib%sx AS bibx ON bx.id=bibx.id_bibxxx LEFT JOIN bibrec AS b ON b.id=bibx.id_bibrec WHERE bx.tag='%s' AND bx.value='%s'" % (CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD[0:2], CFG_OAI_SET_FIELD, set)
    else:
        query = "SELECT DISTINCT bibx.id_bibrec FROM bib%sx AS bx LEFT JOIN bibrec_bib%sx AS bibx ON bx.id=bibx.id_bibxxx LEFT JOIN bibrec AS b ON b.id=bibx.id_bibrec WHERE bx.tag='%s'" % (CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD)

    if untildate:
        query = query + " AND b.modification_date <= '%s'" % untildate
    if fromdate:
        query = query + " AND b.modification_date >= '%s'" % fromdate

    res = run_sql(query)

    for row in res:
        out_dict[row[0]] = 1
         
    return out_dict.keys()
    
def oaigenresumptionToken():
    "Generates unique ID for resumption token management."

    return md5.new(str(time.time())).hexdigest()


def oaicachein(resumptionToken, sysnos):
    "Stores or adds sysnos in cache.  Input is a string of sysnos separated by commas."

    filename = "%s/RTdata/%s" % (cachedir, resumptionToken)

    fil = open(filename, "w")
    cPickle.dump(sysnos, fil)
    fil.close()
    return 1


def oaicacheout(resumptionToken):
    "Restores string of comma-separated system numbers from cache."
    
    sysnos = []

    filename = "%s/RTdata/%s" % (cachedir, resumptionToken)

    if oaicachestatus(resumptionToken):
        fil = open(filename, "r")
        sysnos = cPickle.load(fil)
        fil.close()
    else:
        return 0
    return sysnos


def oaicacheclean():
    "Removes cached resumptionTokens older than specified"
    
    directory = "%s/RTdata" % cachedir

    files = os.listdir(directory)

    for file_ in files:
        filename = directory + "/" + file_
        # cache entry expires when not modified during a specified period of time
        if ((time.time() - os.path.getmtime(filename)) > CFG_OAI_EXPIRE):
            os.remove(filename)

    return 1


def oaicachestatus(resumptionToken):
    "Checks cache status.  Returns 0 for empty, 1 for full."
    
    filename = "%s/RTdata/%s" % (cachedir, resumptionToken)
    
    if os.path.exists(filename):
        if os.path.getsize(filename) > 0:
            return 1
        else:
            return 0
    else:
        return 0


def get_sets():
    "Returns list of sets."

    out = {}
    row = ['', '']

    query = "SELECT setSpec,setName,setDescription FROM oaiARCHIVE"
    res = run_sql(query)
    for row in res:
        row_bis = [row[0], row[1], row[2]]
        out[row[0]] = row_bis
        
    return out.values()


def parse_args(args=""):
    "Parse input args"

    out_args = {
        "verb"             : "",
        "metadataPrefix"   : "",
        "from"             : "",
        "until"            : "",
        "set"              : "",
        "identifier"       : "",
        "resumptionToken"  : ""
    }

    if args == "" or args is None:
        pass
    else:

        list_of_arguments = args.split('&')

        for item in list_of_arguments:
            keyvalue = item.split('=')
            if len(keyvalue) == 2:
                if (out_args.has_key(keyvalue[0])):
                    if(out_args[keyvalue[0]] != ""):
                        out_args[keyvalue[0]] = "Error"
                    else:
                        out_args[keyvalue[0]] = urllib.unquote(keyvalue[1])
                else:
                    out_args[keyvalue[0]] = urllib.unquote(keyvalue[1])
            else:
                out_args['verb'] = ""

    return out_args

def check_args(arguments):
    "Check OAI arguments"

    out_args = {
        "verb"             : "",
        "metadataPrefix"   : "",
        "from"             : "",
        "until"            : "",
        "set"              : "",
        "identifier"       : "",
        "resumptionToken"  : ""
    }

    out = ""

## principal argument required
#
#
    if verbs.has_key(arguments['verb']):
        pass
    else:
        out = out + oai_error("badVerb", "Illegal OAI verb")

## defined args
#
#
    for param in arguments.keys():
        if out_args.has_key(param):
            pass
        else:
            out = out + oai_error("badArgument", "The request includes illegal arguments")

## unique args
#
#
    for param in arguments.keys():
        if (arguments[param] == "Error"):
            out = out + oai_error("badArgument", "The request includes illegal arguments")

## resumptionToken exclusive
#
#
    if ((arguments['from'] != "" or arguments['until'] != "" or arguments['metadataPrefix'] != "" or arguments['identifier'] != "" or arguments['set'] != "") and arguments['resumptionToken'] != ""):

        out = out + oai_error("badArgument", "The request includes illegal arguments")

## datestamp formats
#
#
    if arguments['from'] != "":
        from_length = len(arguments['from'])
        if check_date(arguments['from'], "T00:00:00Z") == "":
            out = out + oai_error("badArgument", "Bad datestamp format in from")
    else:
        from_length = 0

    if arguments['until'] != "":
        until_length = len(arguments['until'])
        if check_date(arguments['until'], "T23:59:59Z") == "":
            out = out + oai_error("badArgument", "Bad datestamp format in until")
    else:
        until_length = 0 

    if from_length != 0:
        if until_length != 0:
            if from_length != until_length:
                out = out + oai_error("badArgument", "Bad datestamp format")

    if arguments['from'] != "" and arguments['until'] != "" and arguments['from'] > arguments['until']:
        out = out + oai_error("badArgument", "Wrong date")

## Identify exclusive
#
#
    if (arguments['verb'] =="Identify" and (arguments['metadataPrefix'] != "" or arguments['identifier'] != "" or arguments['set'] != "" or arguments['from'] != "" or arguments['until'] != "" or arguments['resumptionToken'] != "")):
        out = out + oai_error("badArgument", "The request includes illegal arguments")

## parameters for GetRecord
#
#
    if arguments['verb'] =="GetRecord" and arguments['identifier'] == "":
        out = out + oai_error("badArgument", "Record identifier missing")

    if arguments['verb'] =="GetRecord" and arguments['metadataPrefix'] == "":
        out = out + oai_error("badArgument", "Missing metadataPrefix")


## parameters for ListRecords and ListIdentifiers
#
#
    if (arguments['verb'] =="ListRecords" or arguments['verb'] =="ListIdentifiers") and (arguments['metadataPrefix'] == "" and arguments['resumptionToken'] == ""):
        out = out + oai_error("badArgument", "Missing metadataPrefix")

## Metadata prefix defined
#
#
    if arguments.has_key('metadataPrefix'):
        if ((arguments['metadataPrefix'] in params['metadataPrefix']) or (params['metadataPrefix'] == "")):
            pass
        else:
            out = out + oai_error("badArgument", "Missing metadataPrefix")

    return out

<commit_msg>Improved speed.<commit_after>## $Id$

## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""OAI interface for CDS Invenio written in Python compliant with OAI-PMH2.0"""

__revision__ = "$Id$"

import cPickle
import string
import os
import re
import urllib
import time
import md5

from invenio.config import \
     CFG_OAI_DELETED_POLICY, \
     CFG_OAI_EXPIRE, \
     CFG_OAI_IDENTIFY_DESCRIPTION, \
     CFG_OAI_ID_FIELD, \
     CFG_OAI_ID_PREFIX, \
     CFG_OAI_LOAD, \
     CFG_OAI_SAMPLE_IDENTIFIER, \
     CFG_OAI_SET_FIELD, \
     cachedir, \
     cdsname, \
     supportemail, \
     weburl

from invenio.oai_repository_config import *
from invenio.dbquery import run_sql
from invenio.search_engine import record_exists

verbs = {
    "Identify"            : [""],
    "ListSets"            : ["resumptionToken"],
    "ListMetadataFormats" : ["resumptionToken"],
    "ListRecords"         : ["resumptionToken"],
    "ListIdentifiers"     : ["resumptionToken"],
    "GetRecord"	          : [""]
}

params = {
    "verb" : ["Identify","ListIdentifiers","ListSets","ListMetadataFormats","ListRecords","GetRecord"],
    "metadataPrefix" : ["","oai_dc","marcxml"],
    "from" :[""],
    "until":[""],
    "set" :[""],
    "identifier": [""]
}

def encode_for_xml(strxml):
    "Encode special chars in string for XML-compliancy."

    if strxml is None:
        return strxml
    else:
        strxml = string.replace(strxml, '&', '&amp;')
        strxml = string.replace(strxml, '<', '&lt;')
        return strxml

def escape_space(strxml):
    "Encode special chars in string for URL-compliancy."

    strxml = string.replace(strxml, ' ', '%20')
    return strxml

def encode_for_url(strxml):
    "Encode special chars in string for URL-compliancy."

    strxml = string.replace(strxml, '%', '%25')
    strxml = string.replace(strxml, ' ', '%20')
    strxml = string.replace(strxml, '?', '%3F')
    strxml = string.replace(strxml, '#', '%23')
    strxml = string.replace(strxml, '=', '%3D')
    strxml = string.replace(strxml, '&', '%26')
    strxml = string.replace(strxml, '/', '%2F')
    strxml = string.replace(strxml, ':', '%3A')
    strxml = string.replace(strxml, ';', '%3B')
    strxml = string.replace(strxml, '+', '%2B')

    return strxml

def oai_header(args, verb):
    "Print OAI header"

    out = ""

    out = out + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "\n"
    out = out + "<OAI-PMH xmlns=\"http://www.openarchives.org/OAI/2.0/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd\">\n"

    out = out + " <responseDate>" + oaigetresponsedate() + "</responseDate>\n"

    if verb:
        out = out + " <request verb=\"%s\">%s</request>\n" % (verb, oaigetrequesturl(args))
        out = out + " <%s>\n" % verb
    else:
        out = out + " <request>%s</request>\n" % (oaigetrequesturl(args))

    return out

def oai_footer(verb):
    "Print OAI footer"

    out = ""

    if verb:
        out = "%s </%s>\n" % (out, verb)
    out = out + "</OAI-PMH>\n"

    return out

def oai_error_header(args, verb):
    "Print OAI header"

    out = ""

###    out = "Content-Type: text/xml\n\n"
    out = out + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "\n"
    out = out + "<OAI-PMH xmlns=\"http://www.openarchives.org/OAI/2.0/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd\">\n"

    out = out + " <responseDate>" + oaigetresponsedate() + "</responseDate>\n"
    out = out + " <request verb=\"%s\">%s</request>\n" % (verb, oaigetrequesturl(args))

    return out

def oai_error_footer(verb):
    "Print OAI footer"

    out  = verb
    out  = "</OAI-PMH>\n"
    return out

def get_field(sysno, field):
    "Gets list of field 'field' for the record with 'sysno' system number."

    out   = []
    digit = field[0:2]

    bibbx = "bib%sx" % digit
    bibx  = "bibrec_bib%sx" % digit
    query = "SELECT bx.value FROM %s AS bx, %s AS bibx WHERE bibx.id_bibrec='%s' AND bx.id=bibx.id_bibxxx AND bx.tag='%s'" % (bibbx, bibx, sysno, field)

    res = run_sql(query)

    for row in res:

        out.append(row[0])

    return out

def utc_to_localtime(date):
    "Convert UTC to localtime"

    ldate = date.split("T")[0]
    ltime = date.split("T")[1]

    lhour   = ltime.split(":")[0]
    lminute = ltime.split(":")[1]
    lsec    = ltime.split(":")[2]

    lyear   = ldate.split("-")[0]
    lmonth  = ldate.split("-")[1]
    lday    = ldate.split("-")[2]
    
    timetoconvert = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.mktime((string.atoi(lyear), string.atoi(lmonth), string.atoi(lday), string.atoi(lhour), string.atoi(lminute), string.atoi(lsec[:-1]), 0, 0, -1)) - time.timezone + (time.daylight)*3600))

    return timetoconvert

def localtime_to_utc(date):
    "Convert localtime to UTC"

    ldate = date.split(" ")[0]
    ltime = date.split(" ")[1]

    lhour   = ltime.split(":")[0]
    lminute = ltime.split(":")[1]
    lsec    = ltime.split(":")[2]

    lyear   = ldate.split("-")[0]
    lmonth  = ldate.split("-")[1]
    lday    = ldate.split("-")[2]

    timetoconvert = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.mktime((string.atoi(lyear), string.atoi(lmonth), string.atoi(lday), string.atoi(lhour), string.atoi(lminute), string.atoi(lsec), 0, 0, -1))))

    return timetoconvert

def get_creation_date(sysno):
    "Returns the creation date of the record 'sysno'."
    out   = ""
    res = run_sql("SELECT DATE_FORMAT(creation_date, '%%Y-%%m-%%d %%H:%%i:%%s') FROM bibrec WHERE id=%s", (sysno,), 1)
    if res[0][0]:
        out = localtime_to_utc(res[0][0])
    return out

def get_modification_date(sysno):
    "Returns the date of last modification for the record 'sysno'."
    out = ""
    res = run_sql("SELECT DATE_FORMAT(modification_date,'%%Y-%%m-%%d %%H:%%i:%%s') FROM bibrec WHERE id=%s", (sysno,), 1)
    if res and res[0][0]:
        out = localtime_to_utc(res[0][0])
    return out

def get_earliest_datestamp():
    "Get earliest datestamp in the database"
    out = ""
    res = run_sql("SELECT MIN(DATE_FORMAT(creation_date,'%%Y-%%m-%%d %%H:%%i:%%s')) FROM bibrec", (), 1)
    if res[0][0]:
        out = localtime_to_utc(res[0][0])
    return out

def check_date(date, dtime="T00:00:00Z"):
    "Check if the date has a correct format"

    if(re.sub("[0123456789\-:TZ]", "", date) == ""):
        if len(date) == 10:
            date = date + dtime
        if len(date) == 20:
            date = utc_to_localtime(date)
        else:
            date = ""
    else:
        date = ""
    
    return date

def print_record(sysno, format='marcxml', record_exists_result=None):
    """Prints record 'sysno' formatted according to 'format'.

    - if record does not exist, return nothing.

    - if record has been deleted and CFG_OAI_DELETED_POLICY is
      'transient' or 'deleted', then return only header, with status
      'deleted'.
      
    - if record has been deleted and CFG_OAI_DELETED_POLICY is 'no',
      then return nothing.
    
    Optional parameter 'record_exists_result' has the value of the result
    of the record_exists(sysno) function (in order not to call that function
    again if already done.)
    """
    
    out = ""

    # sanity check:
    if record_exists_result is not None:
        _record_exists = record_exists_result
    else:
        _record_exists = record_exists(sysno)
        
    if not _record_exists:
        return

    if (format == "dc") or (format == "oai_dc"):
        format = "xd"

    # print record opening tags:
    
    out = out + "  <record>\n"

    if _record_exists == -1: # Deleted?
        if CFG_OAI_DELETED_POLICY == "persistent" or \
               CFG_OAI_DELETED_POLICY == "transient":
            out = out + "    <header status=\"deleted\">\n"
        else:
            return
    else:
        out = out + "   <header>\n"

    for ident in get_field(sysno, CFG_OAI_ID_FIELD):
        out = "%s    <identifier>%s</identifier>\n" % (out, escape_space(ident))
    out = "%s    <datestamp>%s</datestamp>\n" % (out, get_modification_date(sysno))
    for set in get_field(sysno, CFG_OAI_SET_FIELD):
        out = "%s    <setSpec>%s</setSpec>\n" % (out, set)
    out = out + "   </header>\n"

    if _record_exists == -1: # Deleted?
        pass
    else:
        out = out + "   <metadata>\n"

        if format == "marcxml":
            out = out + "    <marc:record xmlns:marc=\"http://www.loc.gov/MARC21/slim\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd\" type=\"Bibliographic\">"
            out = out + "     <marc:leader>00000coc  2200000uu 4500</marc:leader>"
            ## MARC21 and XML formats, possibley OAI -- they are not in "bibfmt" table; so fetch all the data from "bibXXx" tables:

            if format == "marcxml":

                out = "%s     <marc:controlfield tag=\"001\">%d</marc:controlfield>\n" % (out, int(sysno))

            for digit1 in range(0, 10):
                for digit2 in range(0, 10):
                    bibbx = "bib%d%dx" % (digit1, digit2)
                    bibx = "bibrec_bib%d%dx" % (digit1, digit2)
                    query = "SELECT b.tag,b.value,bb.field_number FROM %s AS b, %s AS bb "\
                            "WHERE bb.id_bibrec='%s' AND b.id=bb.id_bibxxx AND b.tag LIKE '%s%%' "\
                            "ORDER BY bb.field_number, b.tag ASC" % (bibbx, bibx, sysno, str(digit1)+str(digit2))
                    res = run_sql(query)
                    field_number_old = -999
                    field_old = ""
                    for row in res:
                        field, value, field_number = row[0], row[1], row[2]
                        ind1, ind2 = field[3], field[4]
                        if ind1 == "_":
                            ind1 = " "
                        if ind2 == "_":
                            ind2 = " "                        
                        # print field tag
                        if field_number != field_number_old or field[:-1] != field_old[:-1]:
                            if format == "marcxml":

                                if field_number_old != -999:
                                    if field_old[0:2] == "00":
                                        out = out + "     </marc:controlfield>\n"
                                    else:
                                        out = out + "     </marc:datafield>\n"

                                if field[0:2] == "00":
                                    out = "%s     <marc:controlfield tag=\"%s\">\n" % (out, encode_for_xml(field[0:3]))
                                else:
                                    out = "%s     <marc:datafield tag=\"%s\" ind1=\"%s\" ind2=\"%s\">\n" % (out, encode_for_xml(field[0:3]), encode_for_xml(ind1).lower(), encode_for_xml(ind2).lower())


                            field_number_old = field_number
                            field_old = field
                        # print subfield value
                        if format == "marcxml":
                            value = encode_for_xml(value)

                            if(field[0:2] == "00"):
                                out = "%s      %s\n" % (out, value)
                            else:
                                out = "%s      <marc:subfield code=\"%s\">%s</marc:subfield>\n" % (out, encode_for_xml(field[-1:]), value)

                               
                        # fetch next subfield
                    # all fields/subfields printed in this run, so close the tag:
                    if (format == "marcxml") and field_number_old != -999:
                        if field_old[0:2] == "00":
                            out = out + "     </marc:controlfield>\n"
                        else:
                            out = out + "     </marc:datafield>\n"
                        
            out = out + "    </marc:record>\n"

        elif format == "xd":
        # XML Dublin Core format, possibly OAI -- select only some bibXXx fields:
            out = out + "       <oaidc:dc xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:oaidc=\"http://www.openarchives.org/OAI/2.0/oai_dc/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\">\n"

            for field_ in get_field(sysno, "041__a"):
                out =  "%s         <dc:language>%s</dc:language>\n" % (out, field_)

            for field_ in get_field(sysno, "100__a"):
                out =  "%s         <dc:creator>%s</dc:creator>\n" % (out, encode_for_xml(field_))
 
            for field_ in get_field(sysno, "700__a"):
                out =  "%s         <dc:creator>%s</dc:creator>\n" % (out, encode_for_xml(field_))

            for field_ in get_field(sysno, "245__a"):
                out =  "%s         <dc:title>%s</dc:title>\n" % (out, encode_for_xml(field_))

            for field_ in get_field(sysno, "111__a"):
                out =  "%s         <dc:title>%s</dc:title>\n" % (out, encode_for_xml(field_))

            for field_ in get_field(sysno, "65017a"):
                out =  "%s         <dc:subject>%s</dc:subject>\n" % (out, encode_for_xml(field_))

            for field_ in get_field(sysno, "8564_u"):
                out =  "%s         <dc:identifier>%s</dc:identifier>\n" % (out, encode_for_xml(escape_space(field_)))
        
            for field_ in get_field(sysno, "520__a"):
                out = "%s         <dc:description>%s</dc:description>\n" % (out, encode_for_xml(field_))

            date = get_creation_date(sysno)
 
            out = "%s         <dc:date>%s</dc:date>\n" % (out, date)
            out = out + "    </oaidc:dc>\n"

    # print record closing tags:
    
        out = out + "   </metadata>\n"

    out = out + "  </record>\n"

    return out

def oailistmetadataformats(args):
    "Generates response to oailistmetadataformats verb."

    arg = parse_args(args)

    out = ""

    flag = 1 # list or not depending on identifier

    if arg['identifier'] != "":

        flag = 0

        sysno = oaigetsysno(arg['identifier'])
        _record_exists = record_exists(sysno)
        if _record_exists == 1 or \
               (_record_exists == -1 and CFG_OAI_DELETED_POLICY != "no"):

            flag = 1

        else:

            out = out + oai_error("idDoesNotExist","invalid record Identifier")
            out = oai_error_header(args, "ListMetadataFormats") + out + oai_error_footer("ListMetadataFormats")
            return out

    if flag:
        out = out + "   <metadataFormat>\n"
        out = out + "    <metadataPrefix>oai_dc</metadataPrefix>\n"
        out = out + "    <schema>http://www.openarchives.org/OAI/1.1/dc.xsd</schema>\n"
        out = out + "    <metadataNamespace>http://purl.org/dc/elements/1.1/</metadataNamespace>\n"
        out = out + "   </metadataFormat>\n"
        out = out + "   <metadataFormat>\n"
        out = out + "    <metadataPrefix>marcxml</metadataPrefix>\n"
        out = out + "    <schema>http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd</schema>\n"
        out = out + "    <metadataNamespace>http://www.loc.gov/MARC21/slim</metadataNamespace>\n"
        out = out + "   </metadataFormat>\n"

    out = oai_header(args, "ListMetadataFormats") + out + oai_footer("ListMetadataFormats")
    return out


def oailistrecords(args):
    "Generates response to oailistrecords verb."

    arg = parse_args(args)
    
    out = ""
    resumptionToken_printed = False
    
    sysnos = []
    sysno  = []
    # check if the resumptionToken did not expire
    if arg['resumptionToken']:
        filename = "%s/RTdata/%s" % (cachedir, arg['resumptionToken'])
        if os.path.exists(filename) == 0:
            out = oai_error("badResumptionToken", "ResumptionToken expired")
            out = oai_error_header(args, "ListRecords") + out + oai_error_footer("ListRecords")
            return out

    if arg['resumptionToken'] != "":
        sysnos = oaicacheout(arg['resumptionToken'])
        arg['metadataPrefix'] = sysnos.pop()
    else:
        sysnos = oaigetsysnolist(arg['set'], arg['from'], arg['until'])

    if len(sysnos) == 0: # noRecordsMatch error
        
        out = out + oai_error("noRecordsMatch", "no records correspond to the request")
        out = oai_error_header(args, "ListRecords") + out + oai_error_footer("ListRecords")
        return out

    i = 0
    for sysno_ in sysnos:
        if sysno_:
            if i >= CFG_OAI_LOAD:          # cache or write?
                if not resumptionToken_printed: # resumptionToken?
                    arg['resumptionToken'] = <PASSWORD>()
                    extdate = oaigetresponsedate(CFG_OAI_EXPIRE)
                    if extdate:
                        out = "%s <resumptionToken expirationDate=\"%s\">%s</resumptionToken>\n" % (out, extdate, arg['resumptionToken'])
                    else:
                        out = "%s <resumptionToken>%s</resumptionToken>\n" % (out, arg['resumptionToken'])
                    resumptionToken_printed = True
                sysno.append(sysno_)
            else:
                _record_exists = record_exists(sysno_)
                if not (_record_exists == -1 and CFG_OAI_DELETED_POLICY == "no"):
                    #Produce output only if record exists and had to be printed
                    i = i + 1 # Increment limit only if record is returned
                    res = print_record(sysno_, arg['metadataPrefix'], _record_exists)
                    if res:
                        out += res

    if i >= CFG_OAI_LOAD:
        oaicacheclean()
        sysno.append(arg['metadataPrefix'])
        oaicachein(arg['resumptionToken'], sysno)

    out = oai_header(args, "ListRecords") + out + oai_footer("ListRecords")
    return out

def oailistsets(args):
    "Lists available sets for OAI metadata harvesting."

    out = ""

    # note: no flow control in ListSets

    sets = get_sets()

    for set_ in sets:

        out = out + "  <set>\n"
        out = "%s    <setSpec>%s</setSpec>\n" % (out, set_[0])
        out = "%s    <setName>%s</setName>\n" % (out, set_[1])
        if set_[2]:
            out = "%s    <setDescription>%s</setDescription>\n" % (out, set_[2])
        out = out + "   </set>\n"

    out = oai_header(args, "ListSets") + out + oai_footer("ListSets")

    return out

    
def oaigetrecord(args):
    """Returns record 'identifier' according to 'metadataPrefix' format for OAI metadata harvesting.
    
    - if record does not exist, return oai_error 'idDoesNotExist'.

    - if record has been deleted and CFG_OAI_DELETED_POLICY is
      'transient' or 'deleted', then return only header, with status
      'deleted'.
      
    - if record has been deleted and CFG_OAI_DELETED_POLICY is 'no',
      then return oai_error 'idDoesNotExist'.
    """
    
    arg = parse_args(args)
    out = ""
    sysno = oaigetsysno(arg['identifier'])
    _record_exists = record_exists(sysno)
    if _record_exists == 1 or \
           (_record_exists == -1 and CFG_OAI_DELETED_POLICY != 'no'):
        datestamp = get_modification_date(sysno)
        out += print_record(sysno, arg['metadataPrefix'], _record_exists)
    else:
        out = out + oai_error("idDoesNotExist", "invalid record Identifier")
        out = oai_error_header(args, "GetRecord") + out + oai_error_footer("GetRecord")
    return out

    out = oai_header(args, "GetRecord") + out + oai_footer("GetRecord")

    return out


def oailistidentifiers(args):
    "Prints OAI response to the ListIdentifiers verb."

    arg = parse_args(args)

    out = ""
    resumptionToken_printed = False
    
    sysno  = []
    sysnos = []

    if arg['resumptionToken']:
        filename = "%s/RTdata/%s" % (cachedir, arg['resumptionToken'])
        if os.path.exists(filename) == 0:
            out = out + oai_error("badResumptionToken", "ResumptionToken expired")
            out = oai_error_header(args, "ListIdentifiers") + out + oai_error_footer("ListIdentifiers")
            return out

    if arg['resumptionToken']:
        sysnos = oaicacheout(arg['resumptionToken'])
    else:
        sysnos = oaigetsysnolist(arg['set'], arg['from'], arg['until'])

    if len(sysnos) == 0: # noRecordsMatch error
        out = out + oai_error("noRecordsMatch", "no records correspond to the request")
        out = oai_error_header(args, "ListIdentifiers") + out + oai_error_footer("ListIdentifiers")
        return out

    i = 0
    for sysno_ in sysnos:
        if sysno_:
            if i >= CFG_OAI_LOAD:           # cache or write?
                if not resumptionToken_printed: # resumptionToken?
                    arg['resumptionToken'] = oaigen<PASSWORD>umptionToken()
                    extdate = oaigetresponsedate(CFG_OAI_EXPIRE)
                    if extdate:
                        out = "%s  <resumptionToken expirationDate=\"%s\">%s</resumptionToken>\n" % (out, extdate, arg['resumptionToken'])
                    else:
                        out = "%s  <resumptionToken>%s</resumptionToken>\n" % (out, arg['resumptionToken'])
                    resumptionToken_printed = True
                sysno.append(sysno_)
            else:
                _record_exists = record_exists(sysno_)
                if (not _record_exists == -1 and CFG_OAI_DELETED_POLICY == "no"): 
                    i = i + 1 # Increment limit only if record is returned
                for ident in get_field(sysno_, CFG_OAI_ID_FIELD):
                    if ident != '':
                        if _record_exists == -1: #Deleted?
                            if CFG_OAI_DELETED_POLICY == "persistent" \
                                   or CFG_OAI_DELETED_POLICY == "transient":
                                out = out + "    <header status=\"deleted\">\n"
                        else:
                            out = out + "    <header>\n"
                        out = "%s      <identifier>%s</identifier>\n" % (out, escape_space(ident))
                        out = "%s      <datestamp>%s</datestamp>\n" % (out, get_modification_date(oaigetsysno(ident)))
                        for set in get_field(sysno_, CFG_OAI_SET_FIELD):
                            out = "%s      <setSpec>%s</setSpec>\n" % (out, set)
                        out = out + "    </header>\n"

    if i >= CFG_OAI_LOAD:
        oaicacheclean() # clean cache from expired resumptionTokens
        oaicachein(arg['resumptionToken'], sysno)

    out = oai_header(args, "ListIdentifiers") + out + oai_footer("ListIdentifiers")

    return out


def oaiidentify(args):
    "Generates response to oaiidentify verb."
        
    out = ""

    repositoryname        = "  <repositoryName>" + cdsname + "</repositoryName>\n"
    baseurl               = "  <baseURL>%s/oai2d/</baseURL>\n" % weburl
    protocolversion       = "  <protocolVersion>2.0</protocolVersion>\n"
    adminemail            = "  <adminEmail>%s</adminEmail>\n" % supportemail
    earliestdst		  = "  <earliestDatestamp>%s</earliestDatestamp>\n" % get_earliest_datestamp()               
    deletedrecord         = "  <deletedRecord>%s</deletedRecord>\n" % CFG_OAI_DELETED_POLICY
    repositoryidentifier  = "%s" % CFG_OAI_ID_PREFIX
    sampleidentifier      = CFG_OAI_SAMPLE_IDENTIFIER
    identifydescription   = CFG_OAI_IDENTIFY_DESCRIPTION + "\n"

    out = out + repositoryname
    out = out + baseurl
    out = out + protocolversion
    out = out + adminemail
    out = out + earliestdst
    out = out + deletedrecord
    out = out + "  <granularity>YYYY-MM-DDThh:mm:ssZ</granularity>\n"
    #    print "  <compression></compression>\n"
    out = out + CFG_OAI_IDENTIFY_DESCRIPTION

    out = oai_header(args, "Identify") + out + oai_footer("Identify")

    return out

     
def oaigetrequesturl(args):
    "Generates requesturl tag for OAI."

    # re_amp = re.compile('&')

    requesturl = weburl + "/" + "oai2d/"# + "?" + re_amp.sub("&amp;", args)

    return requesturl

def oaigetresponsedate(delay=0):
    "Generates responseDate tag for OAI."
    
    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() + delay))


def oai_error(code, msg):
    "OAI error occured"

    return "<error code=\"%s\">%s</error>\n" % (code, msg)


def oaigetsysno(identifier):
    "Returns the first database BIB ID for the OAI identifier 'identifier', if it exists."
    sysno = None
    if identifier:
        query = "SELECT DISTINCT(bb.id_bibrec) FROM bib%sx AS bx, bibrec_bib%sx AS bb WHERE bx.tag='%s' AND bb.id_bibxxx=bx.id AND bx.value='%s'" % (CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD, identifier)
        res = run_sql(query)
        for row in res:
            sysno = row[0]
    return sysno


def oaigetsysnolist(set, fromdate, untildate):
    "Returns list of system numbers for the OAI set 'set', modified from 'date_from' until 'date_until'."

    out_dict = {} # dict to hold list of out sysnos as its keys

    if set:
        query = "SELECT DISTINCT bibx.id_bibrec FROM bib%sx AS bx LEFT JOIN bibrec_bib%sx AS bibx ON bx.id=bibx.id_bibxxx LEFT JOIN bibrec AS b ON b.id=bibx.id_bibrec WHERE bx.tag='%s' AND bx.value='%s'" % (CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD[0:2], CFG_OAI_SET_FIELD, set)
    else:
        query = "SELECT DISTINCT bibx.id_bibrec FROM bib%sx AS bx LEFT JOIN bibrec_bib%sx AS bibx ON bx.id=bibx.id_bibxxx LEFT JOIN bibrec AS b ON b.id=bibx.id_bibrec WHERE bx.tag='%s'" % (CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD[0:2], CFG_OAI_ID_FIELD)

    if untildate:
        query = query + " AND b.modification_date <= '%s'" % untildate
    if fromdate:
        query = query + " AND b.modification_date >= '%s'" % fromdate

    res = run_sql(query)

    for row in res:
        out_dict[row[0]] = 1
         
    return out_dict.keys()
    
def oaigenresumptionToken():
    "Generates unique ID for resumption token management."

    return md5.new(str(time.time())).hexdigest()


def oaicachein(resumptionToken, sysnos):
    "Stores or adds sysnos in cache.  Input is a string of sysnos separated by commas."

    filename = "%s/RTdata/%s" % (cachedir, resumptionToken)

    fil = open(filename, "w")
    cPickle.dump(sysnos, fil)
    fil.close()
    return 1


def oaicacheout(resumptionToken):
    "Restores string of comma-separated system numbers from cache."
    
    sysnos = []

    filename = "%s/RTdata/%s" % (cachedir, resumptionToken)

    if oaicachestatus(resumptionToken):
        fil = open(filename, "r")
        sysnos = cPickle.load(fil)
        fil.close()
    else:
        return 0
    return sysnos


def oaicacheclean():
    "Removes cached resumptionTokens older than specified"
    
    directory = "%s/RTdata" % cachedir

    files = os.listdir(directory)

    for file_ in files:
        filename = directory + "/" + file_
        # cache entry expires when not modified during a specified period of time
        if ((time.time() - os.path.getmtime(filename)) > CFG_OAI_EXPIRE):
            os.remove(filename)

    return 1


def oaicachestatus(resumptionToken):
    "Checks cache status.  Returns 0 for empty, 1 for full."
    
    filename = "%s/RTdata/%s" % (cachedir, resumptionToken)
    
    if os.path.exists(filename):
        if os.path.getsize(filename) > 0:
            return 1
        else:
            return 0
    else:
        return 0


def get_sets():
    "Returns list of sets."

    out = {}
    row = ['', '']

    query = "SELECT setSpec,setName,setDescription FROM oaiARCHIVE"
    res = run_sql(query)
    for row in res:
        row_bis = [row[0], row[1], row[2]]
        out[row[0]] = row_bis
        
    return out.values()


def parse_args(args=""):
    "Parse input args"

    out_args = {
        "verb"             : "",
        "metadataPrefix"   : "",
        "from"             : "",
        "until"            : "",
        "set"              : "",
        "identifier"       : "",
        "resumptionToken"  : ""
    }

    if args == "" or args is None:
        pass
    else:

        list_of_arguments = args.split('&')

        for item in list_of_arguments:
            keyvalue = item.split('=')
            if len(keyvalue) == 2:
                if (out_args.has_key(keyvalue[0])):
                    if(out_args[keyvalue[0]] != ""):
                        out_args[keyvalue[0]] = "Error"
                    else:
                        out_args[keyvalue[0]] = urllib.unquote(keyvalue[1])
                else:
                    out_args[keyvalue[0]] = urllib.unquote(keyvalue[1])
            else:
                out_args['verb'] = ""

    return out_args

def check_args(arguments):
    "Check OAI arguments"

    out_args = {
        "verb"             : "",
        "metadataPrefix"   : "",
        "from"             : "",
        "until"            : "",
        "set"              : "",
        "identifier"       : "",
        "resumptionToken"  : ""
    }

    out = ""

## principal argument required
#
#
    if verbs.has_key(arguments['verb']):
        pass
    else:
        out = out + oai_error("badVerb", "Illegal OAI verb")

## defined args
#
#
    for param in arguments.keys():
        if out_args.has_key(param):
            pass
        else:
            out = out + oai_error("badArgument", "The request includes illegal arguments")

## unique args
#
#
    for param in arguments.keys():
        if (arguments[param] == "Error"):
            out = out + oai_error("badArgument", "The request includes illegal arguments")

## resumptionToken exclusive
#
#
    if ((arguments['from'] != "" or arguments['until'] != "" or arguments['metadataPrefix'] != "" or arguments['identifier'] != "" or arguments['set'] != "") and arguments['resumptionToken'] != ""):

        out = out + oai_error("badArgument", "The request includes illegal arguments")

## datestamp formats
#
#
    if arguments['from'] != "":
        from_length = len(arguments['from'])
        if check_date(arguments['from'], "T00:00:00Z") == "":
            out = out + oai_error("badArgument", "Bad datestamp format in from")
    else:
        from_length = 0

    if arguments['until'] != "":
        until_length = len(arguments['until'])
        if check_date(arguments['until'], "T23:59:59Z") == "":
            out = out + oai_error("badArgument", "Bad datestamp format in until")
    else:
        until_length = 0 

    if from_length != 0:
        if until_length != 0:
            if from_length != until_length:
                out = out + oai_error("badArgument", "Bad datestamp format")

    if arguments['from'] != "" and arguments['until'] != "" and arguments['from'] > arguments['until']:
        out = out + oai_error("badArgument", "Wrong date")

## Identify exclusive
#
#
    if (arguments['verb'] =="Identify" and (arguments['metadataPrefix'] != "" or arguments['identifier'] != "" or arguments['set'] != "" or arguments['from'] != "" or arguments['until'] != "" or arguments['resumptionToken'] != "")):
        out = out + oai_error("badArgument", "The request includes illegal arguments")

## parameters for GetRecord
#
#
    if arguments['verb'] =="GetRecord" and arguments['identifier'] == "":
        out = out + oai_error("badArgument", "Record identifier missing")

    if arguments['verb'] =="GetRecord" and arguments['metadataPrefix'] == "":
        out = out + oai_error("badArgument", "Missing metadataPrefix")


## parameters for ListRecords and ListIdentifiers
#
#
    if (arguments['verb'] =="ListRecords" or arguments['verb'] =="ListIdentifiers") and (arguments['metadataPrefix'] == "" and arguments['resumptionToken'] == ""):
        out = out + oai_error("badArgument", "Missing metadataPrefix")

## Metadata prefix defined
#
#
    if arguments.has_key('metadataPrefix'):
        if ((arguments['metadataPrefix'] in params['metadataPrefix']) or (params['metadataPrefix'] == "")):
            pass
        else:
            out = out + oai_error("badArgument", "Missing metadataPrefix")

    return out

def oai_profile():
    """
    Runs a benchmark
    """
    #oailistrecords('set=&from=&metadataPrefix=oai_dc&verb=ListRecords&resumptionToken=&identifier=&until=')
    #oailistrecords('set=&from=&metadataPrefix=marcxml&verb=ListRecords&resumptionToken=&identifier=&until=')
    oailistidentifiers('set=&from=&metadataPrefix=oai_dc&verb=ListIdentifiers&resumptionToken=&identifier=&until=')

    return 

if __name__ == "__main__":   
    import profile
    import pstats
    profile.run('oai_profile()', "oai_profile")
    p = pstats.Stats("oai_profile")
    p.strip_dirs().sort_stats("cumulative").print_stats()
{'fraction_non_alphanumeric': 0.14242318382721486, 'fraction_numerical': 0.011269916007229758, 'mean_word_length': 2.2258696717295443, 'pattern_counts': {'":': 57, '<': 257, '<?xml version=': 4, '>': 269, 'https://': 0, 'lorem ipsum': 0, 'www.': 38, 'xml': 141}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6122048', 'n_tokens_mistral': 22457, 'n_tokens_neox': 21291, 'n_words': 4866}