<commit_before>package se.kb.libris.whelks.importers

import groovy.xml.StreamingMarkupBuilder
import groovy.util.slurpersupport.GPathResult
import groovy.util.logging.Slf4j as Log

import java.text.*

import se.kb.libris.whelks.*
import se.kb.libris.whelks.exception.*
import se.kb.libris.util.marc.*
import se.kb.libris.util.marc.io.*
import se.kb.libris.conch.converter.MarcJSONConverter

@Log
class OAIPMHImporter {

    Whelk whelk
    String resource
    int nrImported = 0
    int nrDeleted = 0
    long startTime = 0
    boolean picky = true

    OAIPMHImporter(Whelk toWhelk, String fromResource) {
        this.whelk = toWhelk
        this.resource = fromResource
    }

    int doImport(Date from = null, int nrOfDocs = -1, boolean picky = true) {
        getAuthentication()
        this.picky = picky
        String urlString = "http://data.libris.kb.se/"+this.resource+"/oaipmh/?verb=ListRecords&metadataPrefix=marcxml"
        if (from) {
            urlString = urlString + "&from=" + from.format("yyyy-MM-dd'T'HH:mm:ss'Z'")
        }
        startTime = System.currentTimeMillis()
        log.info("Harvesting OAIPMH data from $urlString. Pickymode: $picky")
        URL url = new URL(urlString)
        String resumptionToken = harvest(url)
        log.debug("resumptionToken: $resumptionToken")
        while (resumptionToken && (nrOfDocs == -1 || nrImported <  nrOfDocs)) {
            url = new URL("http://data.libris.kb.se/" + this.resource + "/oaipmh/?verb=ListRecords&resumptionToken=" + resumptionToken)
            resumptionToken = harvest(url)
            log.debug("resumptionToken: $resumptionToken")
        }
        return nrImported
    }


    String harvest(URL url) {
        long loadStartTime = System.currentTimeMillis()
        def xmlString = normalizeString(url.text)
        long meanTime = System.currentTimeMillis()
        def OAIPMH
        try {
            OAIPMH = new XmlSlurper(false,false).parseText(xmlString)
        } catch (org.xml.sax.SAXParseException spe) {
            log.error("Failed to parse XML: $xmlString")
            throw spe
        }
        def documents = []
        OAIPMH.ListRecords.record.each {
            def mdrecord = createString(it.metadata.record)
            if (mdrecord) {
                MarcRecord record = MarcXmlRecordReader.fromXml(mdrecord)

                String id = record.getControlfields("001").get(0).getData()
                String jsonRec = MarcJSONConverter.toJSONString(record)

                def links = new HashSet<Link>()
                def tags = new HashSet<Tag>()
                if (it.header.setSpec) {
                    for (sS in it.header.setSpec) {
                        if (sS.toString().startsWith("authority:")) {
                            def authURI = new URI("/auth/" + sS.toString().substring(10))
                            links.add(new Link(authURI, "auth"))
                        }
                        if (sS.toString().startsWith("location:")) {
                            def locationURI = new URI("location")
                            tags.add(new Tag(locationURI, sS.toString().substring(9)))
                        }
                        if (sS.toString().startsWith("bibid:")) {
                            def bibURI = new URI("/bib/" + sS.toString().substring(6))
                            docs.add(new Link(bibURI, "bib"))
                        }
                    }
                }
                def doc
                try {
                    doc = whelk.createDocument(jsonRec.getBytes("UTF-8"), ["identifier":new URI("/"+this.resource+"/"+id),"contentType":"application/x-marc-json", "links": links, "tags": tags])
                } catch (Exception e) {
                    log.error("Failed! (${e.message}) for :\n$mdrecord")
                    if (picky) {
                        throw e
                    }<commit_msg>Updated OAIPMHImporter to work with new document String identifier<commit_after>package se.kb.libris.whelks.importers

import groovy.xml.StreamingMarkupBuilder
import groovy.util.slurpersupport.GPathResult
import groovy.util.logging.Slf4j as Log

import java.text.*

import se.kb.libris.whelks.*
import se.kb.libris.whelks.exception.*
import se.kb.libris.util.marc.*
import se.kb.libris.util.marc.io.*
import se.kb.libris.conch.converter.MarcJSONConverter

@Log
class OAIPMHImporter {

    Whelk whelk
    String resource
    int nrImported = 0
    int nrDeleted = 0
    long startTime = 0
    boolean picky = true

    OAIPMHImporter(Whelk toWhelk, String fromResource) {
        this.whelk = toWhelk
        this.resource = fromResource
    }

    int doImport(Date from = null, int nrOfDocs = -1, boolean picky = true) {
        getAuthentication()
        this.picky = picky
        String urlString = "http://data.libris.kb.se/"+this.resource+"/oaipmh/?verb=ListRecords&metadataPrefix=marcxml"
        if (from) {
            urlString = urlString + "&from=" + from.format("yyyy-MM-dd'T'HH:mm:ss'Z'")
        }
        startTime = System.currentTimeMillis()
        log.info("Harvesting OAIPMH data from $urlString. Pickymode: $picky")
        URL url = new URL(urlString)
        String resumptionToken = harvest(url)
        log.debug("resumptionToken: $resumptionToken")
        while (resumptionToken && (nrOfDocs == -1 || nrImported <  nrOfDocs)) {
            url = new URL("http://data.libris.kb.se/" + this.resource + "/oaipmh/?verb=ListRecords&resumptionToken=" + resumptionToken)
            resumptionToken = harvest(url)
            log.debug("resumptionToken: $resumptionToken")
        }
        return nrImported
    }


    String harvest(URL url) {
        long loadStartTime = System.currentTimeMillis()
        def xmlString = normalizeString(url.text)
        long meanTime = System.currentTimeMillis()
        def OAIPMH
        try {
            OAIPMH = new XmlSlurper(false,false).parseText(xmlString)
        } catch (org.xml.sax.SAXParseException spe) {
            log.error("Failed to parse XML: $xmlString")
            throw spe
        }
        def documents = []
        OAIPMH.ListRecords.record.each {
            def mdrecord = createString(it.metadata.record)
            if (mdrecord) {
                MarcRecord record = MarcXmlRecordReader.fromXml(mdrecord)

                String id = record.getControlfields("001").get(0).getData()
                String jsonRec = MarcJSONConverter.toJSONString(record)

                def links = new HashSet<Link>()
                def tags = new HashSet<Tag>()
                if (it.header.setSpec) {
                    for (sS in it.header.setSpec) {
                        if (sS.toString().startsWith("authority:")) {
                            def authURI = new URI("/auth/" + sS.toString().substring(10))
                            links.add(new Link(authURI, "auth"))
                        }
                        if (sS.toString().startsWith("location:")) {
                            def locationURI = new URI("location")
                            tags.add(new Tag(locationURI, sS.toString().substring(9)))
                        }
                        if (sS.toString().startsWith("bibid:")) {
                            def bibURI = new URI("/bib/" + sS.toString().substring(6))
                            docs.add(new Link(bibURI, "bib"))
                        }
                    }
                }
                def doc
                try {
                    doc = whelk.createDocument(jsonRec.getBytes("UTF-8"), ["identifier":"/"+this.resource+"/"+id,"contentType":"application/x-marc-json", "links": links, "tags": tags])
                } catch (Exception e) {
                    log.error("Failed! (${e.message}) for :\n$mdrecord")
                    if (picky) {
                        throw e
                    }
{'fraction_non_alphanumeric': 0.1165691015525579, 'fraction_numerical': 0.003817765334690761, 'mean_word_length': 1.8269784172661871, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7059196', 'n_tokens_mistral': 2376, 'n_tokens_neox': 2237, 'n_words': 441}