<commit_before>package se.kb.libris.whelks.plugin

import se.kb.libris.whelks.*

import groovy.util.logging.Slf4j as Log

import groovy.json.*

@Log
class MarcCrackerIndexFormatConverter implements IndexFormatConverter {

    String id = this.class.name
    boolean enabled = true
    def marcmap 

    MarcCrackerIndexFormatConverter() { InputStream is = this.getClass().getClassLoader().getResourceAsStream("marcmap.json")
        this.marcmap = new JsonSlurper().parse(is.newReader())
    }

    def expandField(ctrlfield, columns) {
        def l = [:]
        def propref
        for (def column : columns) {
            if (propref != column.propRef) {
                l[column.propRef] = ""
            }
            if (column.length == 1) {
                l[column.propRef] += ctrlfield[column.offset]
            } else {
                l[column.propRef] += ctrlfield[(column.offset)..(column.offset+column.length-1)]
            }
            propref = column.propRef
        }
        println "expandField l: $l"
        return l

    }
    

    @Override
    Document convert(Document doc) {
        def json = new JsonSlurper().parseText(doc.dataAsString)
        def leader = json.leader

        def l = expandField(leader, marcmap.bib."000".fixmaps[0].columns)

        json.leader = ["subfields": l.collect {key, value -> [(key):value]}]

        def mrtbl = l['typeOfRecord'] + l['bibLevel']

        json.fields.eachWithIndex() { it, pos ->
            it.each { fkey, fvalue ->
                if (fkey.startsWith("00")) {
                    marcmap.bib.each { key, value ->
                        if (fkey == key) {
                            try {
                                value.fixmaps.each { fm ->
                                    if ((!fm.matchRecTypeBibLevel && fm.matchKeys.contains(l['typeOfRecord'])) || fm.matchRecTypeBibLevel.contains(mrtbl)) {
                                        json.fields[pos] = [(fkey):["subfields": expandField(fvalue, fm.columns).collect {k, v -> [(k):v]} ]]
                                    }
                                } catch (groovy.lang.MissingPropertyException mpe) { mpe.printStackTrace() }
                            }
                        }
                    }
                }
            }
        }

        def builder = new JsonBuilder(json)
        doc.withData(builder.toString())

        return doc
    }


    void enable() { this.enabled = true }
    void disable() { this.enabled = false }
}<commit_msg>Removed trace output<commit_after>package se.kb.libris.whelks.plugin

import se.kb.libris.whelks.*

import groovy.util.logging.Slf4j as Log

import groovy.json.*

@Log
class MarcCrackerIndexFormatConverter implements IndexFormatConverter {

    String id = this.class.name
    boolean enabled = true
    def marcmap 

    MarcCrackerIndexFormatConverter() { InputStream is = this.getClass().getClassLoader().getResourceAsStream("marcmap.json")
        this.marcmap = new JsonSlurper().parse(is.newReader())
    }

    def expandField(ctrlfield, columns) {
        def l = [:]
        def propref
        for (def column : columns) {
            if (propref != column.propRef) {
                l[column.propRef] = ""
            }
            if (column.length == 1) {
                l[column.propRef] += ctrlfield[column.offset]
            } else {
                l[column.propRef] += ctrlfield[(column.offset)..(column.offset+column.length-1)]
            }
            propref = column.propRef
        }
        return l

    }
    

    @Override
    Document convert(Document doc) {
        def json = new JsonSlurper().parseText(doc.dataAsString)
        def leader = json.leader

        def l = expandField(leader, marcmap.bib."000".fixmaps[0].columns)

        json.leader = ["subfields": l.collect {key, value -> [(key):value]}]

        def mrtbl = l['typeOfRecord'] + l['bibLevel']

        json.fields.eachWithIndex() { it, pos ->
            it.each { fkey, fvalue ->
                if (fkey.startsWith("00")) {
                    marcmap.bib.each { key, value ->
                        if (fkey == key) {
                            try {
                                value.fixmaps.each { fm ->
                                    if ((!fm.matchRecTypeBibLevel && fm.matchKeys.contains(l['typeOfRecord'])) || fm.matchRecTypeBibLevel.contains(mrtbl)) {
                                        json.fields[pos] = [(fkey):["subfields": expandField(fvalue, fm.columns).collect {k, v -> [(k):v]} ]]
                                    }
                                } catch (groovy.lang.MissingPropertyException mpe) { }
                            }
                        }
                    }
                }
            }
        }

        def builder = new JsonBuilder(json)
        doc.withData(builder.toString())

        return doc
    }


    void enable() { this.enabled = true }
    void disable() { this.enabled = false }
}
{'fraction_non_alphanumeric': 0.11990359509941756, 'fraction_numerical': 0.003615183771841735, 'mean_word_length': 1.555156490507953, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3319011', 'n_tokens_mistral': 1461, 'n_tokens_neox': 1362, 'n_words': 263}