<commit_before>package se.kb.libris

import groovy.util.logging.Log4j2 as Log
import whelk.Document
import whelk.JsonLd
import whelk.Whelk
import whelk.component.DocumentNormalizer
import whelk.exception.InvalidQueryException
import whelk.filter.BlankNodeLinker
import whelk.filter.LanguageLinker
import whelk.util.DocumentUtil
import whelk.util.DocumentUtil.Remove

import static whelk.JsonLd.GRAPH_KEY
import static whelk.JsonLd.ID_KEY
import static whelk.JsonLd.TYPE_KEY
import static whelk.JsonLd.asList
import static whelk.util.DocumentUtil.traverse

/*
TODO: add support for linking blank nodes based on owl:hasKey
example: 
:Concept a owl:Class;
    rdfs:label "Concept"@en, "Koncept"@sv;
    rdfs:subClassOf :Identity;
    owl:equivalentClass skos:Concept;
    owl:hasKey (:code :prefLabel :inScheme) .

(only hasKey defined in vocab at the moment)
 */

@Log
class Normalizers {
    
    static DocumentNormalizer nullRemover() {
        return { Document doc ->
            traverse(doc.data, { value, path ->
                if (value == null) {
                    new Remove()
                }
            })
        }
    }

    static DocumentNormalizer language(Whelk whelk) {
        LanguageLinker linker = new LanguageLinker()
        loadDefinitions(linker, whelk)

        return { Document doc ->
            linker.linkAll(doc.data, 'associatedLanguage')
            linker.linkAll(doc.data, 'language')
        }
    }

    /**
     * Link blank nodes based on "heuristic identifiers"
     * e.g. { "@type": "Role", "label": "Þýðandi"} matches https://id.kb.se/relator/trl on prefLabelByLang.is
     * 
     * For all types that have :category :heuristicIdentity in vocab:
     * Link all blank nodes with that @type that match on a property that has :category :heuristicIdentifier.
     * Only check blank nodes in properties where @type is in range (range or rangeIncludes).
     */
    static Collection<DocumentNormalizer> heuristicLinkers(Whelk whelk) {
        def properties = whelk.jsonld.getCategoryMembers('heuristicIdentifier').collect()
        properties = properties + properties.findResults {(String) whelk.jsonld.langContainerAlias[it] }
        
        whelk.jsonld.getCategoryMembers('heuristicIdentity').collect{ type ->
            BlankNodeLinker linker = new BlankNodeLinker(type, properties)
            loadDefinitions(linker, whelk)

            Set<String> inRange = whelk.jsonld.getInRange(type)
            return (DocumentNormalizer) { doc ->
                linker.linkAll(doc.data, inRange)
            }
        }
    }

    static DocumentNormalizer identifiedBy() {
        def OBSOLETE_TYPE_NOTES = [
                'ansi'    : 'Ansi',
                'doi'     : 'DOI',
                'hdl'     : 'Hdl',
                'isan'    : 'ISAN',
                'isni'    : 'ISNI',
                'iso'     : 'ISO',
                'istc'    : 'ISTC',
                'iswc'    : 'ISWC',
                'orcid'   : 'ORCID',
                'urn'     : 'URN',
                'viaf'    : 'VIAF',
                'wikidata': 'WikidataID',
                // already typed in Marc bib 028 ind1:
                //if $2=matrix-number, then I - identifiedBy - MatrixNumber;
                //if $2=music-plate, then I - identifiedBy - MusicPlate;
                //if $2=music-publisher, then I - identifiedBy - MusicPublisherNumber;
                //if $2=videorecording-identifer, then I - identifiedBy - VideoRecordingNumber;
        ]
        
        return { Document doc ->
            def (_record, thing) = doc.data[GRAPH_KEY]
            thing.identifiedBy?.with {
                asList(it).forEach { Map id ->
                    id.typeNote?.with { String tn -> 
                        OBSOLETE_TYPE_NOTES[tn.toLowerCase()] <commit_msg>add uri to obsolete type notes<commit_after>package se.kb.libris

import groovy.util.logging.Log4j2 as Log
import whelk.Document
import whelk.JsonLd
import whelk.Whelk
import whelk.component.DocumentNormalizer
import whelk.exception.InvalidQueryException
import whelk.filter.BlankNodeLinker
import whelk.filter.LanguageLinker
import whelk.util.DocumentUtil
import whelk.util.DocumentUtil.Remove

import static whelk.JsonLd.GRAPH_KEY
import static whelk.JsonLd.ID_KEY
import static whelk.JsonLd.TYPE_KEY
import static whelk.JsonLd.asList
import static whelk.util.DocumentUtil.traverse

/*
TODO: add support for linking blank nodes based on owl:hasKey
example: 
:Concept a owl:Class;
    rdfs:label "Concept"@en, "Koncept"@sv;
    rdfs:subClassOf :Identity;
    owl:equivalentClass skos:Concept;
    owl:hasKey (:code :prefLabel :inScheme) .

(only hasKey defined in vocab at the moment)
 */

@Log
class Normalizers {
    
    static DocumentNormalizer nullRemover() {
        return { Document doc ->
            traverse(doc.data, { value, path ->
                if (value == null) {
                    new Remove()
                }
            })
        }
    }

    static DocumentNormalizer language(Whelk whelk) {
        LanguageLinker linker = new LanguageLinker()
        loadDefinitions(linker, whelk)

        return { Document doc ->
            linker.linkAll(doc.data, 'associatedLanguage')
            linker.linkAll(doc.data, 'language')
        }
    }

    /**
     * Link blank nodes based on "heuristic identifiers"
     * e.g. { "@type": "Role", "label": "Þýðandi"} matches https://id.kb.se/relator/trl on prefLabelByLang.is
     * 
     * For all types that have :category :heuristicIdentity in vocab:
     * Link all blank nodes with that @type that match on a property that has :category :heuristicIdentifier.
     * Only check blank nodes in properties where @type is in range (range or rangeIncludes).
     */
    static Collection<DocumentNormalizer> heuristicLinkers(Whelk whelk) {
        def properties = whelk.jsonld.getCategoryMembers('heuristicIdentifier').collect()
        properties = properties + properties.findResults {(String) whelk.jsonld.langContainerAlias[it] }
        
        whelk.jsonld.getCategoryMembers('heuristicIdentity').collect{ type ->
            BlankNodeLinker linker = new BlankNodeLinker(type, properties)
            loadDefinitions(linker, whelk)

            Set<String> inRange = whelk.jsonld.getInRange(type)
            return (DocumentNormalizer) { doc ->
                linker.linkAll(doc.data, inRange)
            }
        }
    }

    static DocumentNormalizer identifiedBy() {
        def OBSOLETE_TYPE_NOTES = [
                'ansi'    : 'Ansi',
                'doi'     : 'DOI',
                'hdl'     : 'Hdl',
                'isan'    : 'ISAN',
                'isni'    : 'ISNI',
                'iso'     : 'ISO',
                'istc'    : 'ISTC',
                'iswc'    : 'ISWC',
                'orcid'   : 'ORCID',
                'uri'     : 'URI',
                'urn'     : 'URN',
                'viaf'    : 'VIAF',
                'wikidata': 'WikidataID',
                // already typed in Marc bib 028 ind1:
                //if $2=matrix-number, then I - identifiedBy - MatrixNumber;
                //if $2=music-plate, then I - identifiedBy - MusicPlate;
                //if $2=music-publisher, then I - identifiedBy - MusicPublisherNumber;
                //if $2=videorecording-identifer, then I - identifiedBy - VideoRecordingNumber;
        ]
        
        return { Document doc ->
            def (_record, thing) = doc.data[GRAPH_KEY]
            thing.identifiedBy?.with {
                asList(it).forEach { Map id ->
                    id.typeNote?.with { String tn -> 
                        OBSOLETE_TYPE_NOTES[tn.toLowerCase()] 
{'fraction_non_alphanumeric': 0.1007569825110937, 'fraction_numerical': 0.0026102845210127902, 'mean_word_length': 2.066426570628251, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6343584', 'n_tokens_mistral': 2389, 'n_tokens_neox': 2216, 'n_words': 537}