<commit_before>package se.kb.libris.whelks.api

import groovy.util.logging.Slf4j as Log

import org.codehaus.jackson.map.*
import org.codehaus.jackson.map.SerializationConfig.Feature

import java.util.concurrent.*

import se.kb.libris.whelks.*
import se.kb.libris.whelks.component.*
import se.kb.libris.whelks.exception.*
import se.kb.libris.whelks.importers.*
import se.kb.libris.whelks.plugin.*

import se.kb.libris.conch.Tools

@Log
@Deprecated
class TransferOperator extends AbstractOperator implements Plugin {
    String oid = "transfer"

    // Unique for this operator
    String fromStorage = null
    String toStorage = null

    boolean showSpinner = false

    @Override
    void setParameters(Map parameters) {
        super.setParameters(parameters)
        this.fromStorage = parameters.get("fromStorage", null)?.first()
        this.toStorage = parameters.get("toStorage", null)?.first()
        this.showSpinner = parameters.get("showSpinner", false)
        assert fromStorage
        assert toStorage
    }

    void doRun(long startTime) {
        Storage targetStorage = whelk.getStorages().find { it.id == toStorage }
        Storage sourceStorage = whelk.getStorages().find { it.id == fromStorage }
        assert targetStorage, sourceStorage
        List docs = []
        log.info("Transferring data from $fromStorage to $toStorage")
        boolean versioningOriginalSetting = targetStorage.versioning
        targetStorage.versioning = false

        FormatConverter fc = plugins.find { it instanceof FormatConverter && it.requiredContentType == sourceStorage.contentTypes.first() }
        Filter filter = plugins.find { it instanceof Filter && (!fc || fc.resultContentType == it.requiredContentType) }
        if (fc) {
            log.info("Using formatconverter ${fc.id}")
        }

        for (doc in sourceStorage.getAll(dataset)) {
            log.trace("Storing doc ${doc.identifier} with type ${doc.contentType}")
            if (!doc.entry.deleted) {
                if (fc) {
                    if (filter) {
                        docs << filter.filter(fc.convert(doc))
                    } else {
                        docs << fc.convert(doc)
                    }
                } else {
                    if (filter) {
                        docs << filter.filter(doc)
                    } else {
                        docs << doc
                    }
                }
            } else {
                log.warn("Document ${doc.identifier} is deleted. Don't try to add it.")
            }
            runningTime = System.currentTimeMillis() - startTime
            if (count++ % 10000 == 0) {
                targetStorage.bulkStore(docs)
                docs = []
            }
            if (showSpinner) {
                def velocityMsg = "Current velocity: ${count/(runningTime/1000)}."
                Tools.printSpinner("Transferring from ${fromStorage} to ${toStorage}. ${count} documents transferred sofar. $velocityMsg", count)
            }
            if (cancelled) {
                break
            }<commit_msg>Added check for null in transfer<commit_after>package se.kb.libris.whelks.api

import groovy.util.logging.Slf4j as Log

import org.codehaus.jackson.map.*
import org.codehaus.jackson.map.SerializationConfig.Feature

import java.util.concurrent.*

import se.kb.libris.whelks.*
import se.kb.libris.whelks.component.*
import se.kb.libris.whelks.exception.*
import se.kb.libris.whelks.importers.*
import se.kb.libris.whelks.plugin.*

import se.kb.libris.conch.Tools

@Log
@Deprecated
class TransferOperator extends AbstractOperator implements Plugin {
    String oid = "transfer"

    // Unique for this operator
    String fromStorage = null
    String toStorage = null

    boolean showSpinner = false

    @Override
    void setParameters(Map parameters) {
        super.setParameters(parameters)
        this.fromStorage = parameters.get("fromStorage", null)?.first()
        this.toStorage = parameters.get("toStorage", null)?.first()
        this.showSpinner = parameters.get("showSpinner", false)
        assert fromStorage
        assert toStorage
    }

    void doRun(long startTime) {
        Storage targetStorage = whelk.getStorages().find { it.id == toStorage }
        Storage sourceStorage = whelk.getStorages().find { it.id == fromStorage }
        assert targetStorage, sourceStorage
        List docs = []
        log.info("Transferring data from $fromStorage to $toStorage")
        boolean versioningOriginalSetting = targetStorage.versioning
        targetStorage.versioning = false

        FormatConverter fc = plugins.find { it instanceof FormatConverter && it.requiredContentType == sourceStorage.contentTypes.first() }
        Filter filter = plugins.find { it instanceof Filter && (!fc || fc.resultContentType == it.requiredContentType) }
        if (fc) {
            log.info("Using formatconverter ${fc.id}")
        }

        for (doc in sourceStorage.getAll(dataset)) {
            log.trace("Storing doc ${doc.identifier} with type ${doc.contentType}")
            if (doc && !doc.entry.deleted) {
                if (fc) {
                    if (filter) {
                        docs << filter.filter(fc.convert(doc))
                    } else {
                        docs << fc.convert(doc)
                    }
                } else {
                    if (filter) {
                        docs << filter.filter(doc)
                    } else {
                        docs << doc
                    }
                }
            } else {
                log.warn("Document ${doc.identifier} is deleted. Don't try to add it.")
            }
            runningTime = System.currentTimeMillis() - startTime
            if (count++ % 10000 == 0) {
                targetStorage.bulkStore(docs)
                docs = []
            }
            if (showSpinner) {
                def velocityMsg = "Current velocity: ${count/(runningTime/1000)}."
                Tools.printSpinner("Transferring from ${fromStorage} to ${toStorage}. ${count} documents transferred sofar. $velocityMsg", count)
            }
            if (cancelled) {
                break
            }
{'fraction_non_alphanumeric': 0.10438144329896908, 'fraction_numerical': 0.0035438144329896907, 'mean_word_length': 1.942654028436019, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2887666', 'n_tokens_mistral': 1701, 'n_tokens_neox': 1607, 'n_words': 378}