<commit_before>package whelk.converter.marc

import groovy.transform.CompileStatic
import groovy.util.logging.Log4j2 as Log
import org.codehaus.jackson.map.ObjectMapper
import whelk.Document
import whelk.JsonLd
import whelk.Whelk
import whelk.component.PostgreSQLComponent
import whelk.converter.FormatConverter
import whelk.filter.LinkFinder
import whelk.util.PropertyLoader

import java.time.*
import java.time.format.DateTimeFormatter
import java.time.format.DateTimeParseException
import java.time.temporal.Temporal
import java.util.regex.Pattern

import static com.damnhandy.uri.template.UriTemplate.fromTemplate
import static groovy.transform.TypeCheckingMode.SKIP

@Log
class MarcFrameConverter implements FormatConverter {

    ObjectMapper mapper = new ObjectMapper()
    String cfgBase = "ext"
    LinkFinder linkFinder
    JsonLd ld

    protected MarcConversion conversion

    MarcFrameConverter(LinkFinder linkFinder = null, JsonLd ld = null) {
        this.linkFinder = linkFinder
        setLd(ld)
    }

    void setLd(JsonLd ld) {
        this.ld = ld
        if (ld) initialize()
    }

    void initialize() {
        if (conversion) return
        initialize(readConfig("$cfgBase/marcframe.json"))
    }

    void initialize(Map config) {
        def tokenMaps = loadTokenMaps(config.tokenMaps)
        conversion = new MarcConversion(this, config, tokenMaps)
    }

    Map readConfig(String path) {
        return getClass().classLoader.getResourceAsStream(path).withStream {
            mapper.readValue(it, SortedMap)
        }
    }

    Map loadTokenMaps(tokenMaps) {
        def result = [:]
        def maps = [:]
        if (tokenMaps instanceof String) {
            tokenMaps = [tokenMaps]
        }
        if (tokenMaps instanceof List) {
            tokenMaps.each {
                if (it instanceof String) {
                    maps += readConfig("$cfgBase/$it")
                } else {
                    maps += it
                }
            }
        } else {
            maps = tokenMaps
        }
        maps.each { key, src ->
            if (src instanceof String) {
                result[key] = readConfig("$cfgBase/$src")
            } else {
                result[key] = src
            }
        }
        return result
    }

    Map runConvert(Map marcSource, String recordId = null, Map extraData = null) {
        initialize()
        return conversion.convert(marcSource, recordId, extraData)
    }

    Map runRevert(Map data) {
        initialize()
        if (data['@graph']) {
            def entryId = data['@graph'][0]['@id']
            data = JsonLd.frame(entryId, data)
        }
        return conversion.revert(data)
    }

    @Override
    String getResultContentType() { "application/ld+json" }

    @Override
    String getRequiredContentType() { "application/x-marc-json" }

    @Override
    Map convert(Map source, String id) {
        def result = runConvert(source, id, null)
        log.trace("Created frame: $result")
        return result
    }

    Map convert(Map source, String id, Map extraData) {
        def result = runConvert(source, id, extraData)
        log.trace("Created frame: $result")
        return result
    }

}

@Log
class MarcConversion {

    static MARC_CATEGORIES = ['bib', 'auth', 'hold']
    static Map<String, Integer> ENTITY_ORDER = ['?record': 0, '?thing': 1]

    MarcFrameConverter converter
    List<MarcFramePostProcStep> sharedPostProcSteps
    Map<String, MarcRuleSet> marcRuleSets = [:]
    boolean doPostProcessing = true
    boolean flatLinkedForm = true
    boolean keepGroupIds = false
    Map marcTypeMap = [:]
    Map tokenMaps

    private Set missingTerms = [] as Set
    private Set badRepeats = [] as Set

    URI baseUri = Document.BASE_URI

    MarcConversion(MarcFrameConverter converter, Map config, Map tokenMaps) {
        marcTypeMap = config.marcTypeFromTypeOfRecord
        this.tokenMaps = tokenMaps
        this.converter = converter
        //this.baseUri = new URI(config.baseUri ?: '/')
        this.keepGroupIds = config.keepGroupIds == true

        this.sharedPostProcSteps = config.postProcessing.collect {
            parsePostProcStep(it)
        }

        MARC_CATEGORIES.each { marcCat ->
            def marcRuleSet = new MarcRuleSet(this, marcCat)
            marcRuleSets[marcCat] = marcRuleSet
            marcRuleSet.buildHandlers(config)
        }
        addTypeMaps()

        // log.warn? Though, this "should" not happen in prod...
        missingTerms.each {
            System.err.println "Missing: $it.term a $it.type"
        }
        badRepeats.each {
            if (it.term.startsWith(converter.ld.expand('marc:')))
                return
            if (it.shouldRepeat)
                System.err.println "Expected to be repeatable: $it.term"
            else
                System.err.println "Unexpected repeat of: $it.term"
        }
    }

    void checkTerm(String key, String type, boolean repeatable) {
        if (!key || key == '@type' || !converter.ld)
            return
        def terms = converter.ld.vocabIndex.keySet()
        String term = converter.ld.expand((String) key)
        if (!(term in terms)) {
            missingTerms << [term: term, type: type]
        }
        def ctxDfn = converter.ld.context[key]
        def shouldRepeat = ctxDfn instanceof Map && ctxDfn['@container'] == '@set'
        if (repeatable != shouldRepeat) {
            badRepeats << [term: term, shouldRepeat: shouldRepeat]
        }
    }

    MarcFramePostProcStep parsePostProcStep(Map stepDfn) {
        def props = stepDfn.clone()
        for (k in stepDfn.keySet())
            if (k[0] == '_')
                props.remove(k)
        switch (stepDfn.type) {
            case 'FoldLinkedProperty': new FoldLinkedPropertyStep(props); break
            case 'FoldJoinedProperties': new FoldJoinedPropertiesStep(props); break
            case 'MappedProperty': new MappedPropertyStep(props); break
            case 'VerboseRevertData': new VerboseRevertDataStep(props); break
        }
    }

    void addTypeMaps() {
        tokenMaps.typeOfRecord.each { token, typeName ->
            def marcCat = marcTypeMap[token] ?: marcTypeMap['*']
            def marcRuleSet = marcRuleSets[marcCat]
            marcRuleSet.aboutTypeMap['?thing'] << typeName
        }
    }

    String getMarcCategory(String leader) {
        def typeOfRecord = getTypeOfRecord(leader)<commit_msg>Use repeatableTerms from JsonLd in repeatable check<commit_after>package whelk.converter.marc

import groovy.transform.CompileStatic
import groovy.util.logging.Log4j2 as Log
import org.codehaus.jackson.map.ObjectMapper
import whelk.Document
import whelk.JsonLd
import whelk.Whelk
import whelk.component.PostgreSQLComponent
import whelk.converter.FormatConverter
import whelk.filter.LinkFinder
import whelk.util.PropertyLoader

import java.time.*
import java.time.format.DateTimeFormatter
import java.time.format.DateTimeParseException
import java.time.temporal.Temporal
import java.util.regex.Pattern

import static com.damnhandy.uri.template.UriTemplate.fromTemplate
import static groovy.transform.TypeCheckingMode.SKIP

@Log
class MarcFrameConverter implements FormatConverter {

    ObjectMapper mapper = new ObjectMapper()
    String cfgBase = "ext"
    LinkFinder linkFinder
    JsonLd ld

    protected MarcConversion conversion

    MarcFrameConverter(LinkFinder linkFinder = null, JsonLd ld = null) {
        this.linkFinder = linkFinder
        setLd(ld)
    }

    void setLd(JsonLd ld) {
        this.ld = ld
        if (ld) initialize()
    }

    void initialize() {
        if (conversion) return
        initialize(readConfig("$cfgBase/marcframe.json"))
    }

    void initialize(Map config) {
        def tokenMaps = loadTokenMaps(config.tokenMaps)
        conversion = new MarcConversion(this, config, tokenMaps)
    }

    Map readConfig(String path) {
        return getClass().classLoader.getResourceAsStream(path).withStream {
            mapper.readValue(it, SortedMap)
        }
    }

    Map loadTokenMaps(tokenMaps) {
        def result = [:]
        def maps = [:]
        if (tokenMaps instanceof String) {
            tokenMaps = [tokenMaps]
        }
        if (tokenMaps instanceof List) {
            tokenMaps.each {
                if (it instanceof String) {
                    maps += readConfig("$cfgBase/$it")
                } else {
                    maps += it
                }
            }
        } else {
            maps = tokenMaps
        }
        maps.each { key, src ->
            if (src instanceof String) {
                result[key] = readConfig("$cfgBase/$src")
            } else {
                result[key] = src
            }
        }
        return result
    }

    Map runConvert(Map marcSource, String recordId = null, Map extraData = null) {
        initialize()
        return conversion.convert(marcSource, recordId, extraData)
    }

    Map runRevert(Map data) {
        initialize()
        if (data['@graph']) {
            def entryId = data['@graph'][0]['@id']
            data = JsonLd.frame(entryId, data)
        }
        return conversion.revert(data)
    }

    @Override
    String getResultContentType() { "application/ld+json" }

    @Override
    String getRequiredContentType() { "application/x-marc-json" }

    @Override
    Map convert(Map source, String id) {
        def result = runConvert(source, id, null)
        log.trace("Created frame: $result")
        return result
    }

    Map convert(Map source, String id, Map extraData) {
        def result = runConvert(source, id, extraData)
        log.trace("Created frame: $result")
        return result
    }

}

@Log
class MarcConversion {

    static MARC_CATEGORIES = ['bib', 'auth', 'hold']
    static Map<String, Integer> ENTITY_ORDER = ['?record': 0, '?thing': 1]

    MarcFrameConverter converter
    List<MarcFramePostProcStep> sharedPostProcSteps
    Map<String, MarcRuleSet> marcRuleSets = [:]
    boolean doPostProcessing = true
    boolean flatLinkedForm = true
    boolean keepGroupIds = false
    Map marcTypeMap = [:]
    Map tokenMaps

    private Set missingTerms = [] as Set
    private Set badRepeats = [] as Set

    URI baseUri = Document.BASE_URI

    MarcConversion(MarcFrameConverter converter, Map config, Map tokenMaps) {
        marcTypeMap = config.marcTypeFromTypeOfRecord
        this.tokenMaps = tokenMaps
        this.converter = converter
        //this.baseUri = new URI(config.baseUri ?: '/')
        this.keepGroupIds = config.keepGroupIds == true

        this.sharedPostProcSteps = config.postProcessing.collect {
            parsePostProcStep(it)
        }

        MARC_CATEGORIES.each { marcCat ->
            def marcRuleSet = new MarcRuleSet(this, marcCat)
            marcRuleSets[marcCat] = marcRuleSet
            marcRuleSet.buildHandlers(config)
        }
        addTypeMaps()

        // log.warn? Though, this "should" not happen in prod...
        missingTerms.each {
            System.err.println "Missing: $it.term a $it.type"
        }
        badRepeats.each {
            if (it.term.startsWith(converter.ld.expand('marc:')))
                return
            if (it.shouldRepeat)
                System.err.println "Expected to be repeatable: $it.term"
            else
                System.err.println "Unexpected repeat of: $it.term"
        }
    }

    void checkTerm(String key, String type, boolean repeatable) {
        if (!key || key == '@type' || !converter.ld)
            return
        def terms = converter.ld.vocabIndex.keySet()
        String term = converter.ld.expand((String) key)
        if (!(term in terms)) {
            missingTerms << [term: term, type: type]
        }
        def shouldRepeat = key in converter.ld.repeatableTerms
        if (repeatable != shouldRepeat) {
            badRepeats << [term: term, shouldRepeat: shouldRepeat]
        }
    }

    MarcFramePostProcStep parsePostProcStep(Map stepDfn) {
        def props = stepDfn.clone()
        for (k in stepDfn.keySet())
            if (k[0] == '_')
                props.remove(k)
        switch (stepDfn.type) {
            case 'FoldLinkedProperty': new FoldLinkedPropertyStep(props); break
            case 'FoldJoinedProperties': new FoldJoinedPropertiesStep(props); break
            case 'MappedProperty': new MappedPropertyStep(props); break
            case 'VerboseRevertData': new VerboseRevertDataStep(props); break
        }
    }

    void addTypeMaps() {
        tokenMaps.typeOfRecord.each { token, typeName ->
            def marcCat = marcTypeMap[token] ?: marcTypeMap['*']
            def marcRuleSet = marcRuleSets[marcCat]
            marcRuleSet.aboutTypeMap['?thing'] << typeName
        }
    }

    String getMarcCategory(String leader) {
        def typeOfRecord = getTypeOfRecord(leader)
{'fraction_non_alphanumeric': 0.09666899604006522, 'fraction_numerical': 0.0009317493594223154, 'mean_word_length': 2.273189326556544, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1920597', 'n_tokens_mistral': 3851, 'n_tokens_neox': 3707, 'n_words': 854}