<commit_before>package vars.annotation

import java.text.SimpleDateFormat
import java.util.concurrent.BlockingQueue
import java.util.concurrent.TimeUnit
import java.util.concurrent.LinkedBlockingQueue
import org.slf4j.LoggerFactory
import vars.integration.MergeStatusDAO
import org.mbari.vars.integration.MergeEXPDAnnotations
import vars.integration.MergeFunction
import org.mbari.expd.UberDatum

class DatabaseUtility {

    final log = LoggerFactory.getLogger(DatabaseUtility.class)
    final toolBox = new vars.ToolBox()


    def DatabaseUtility() {
    }

    void merge() {

        log.debug("----- Merging Annotations with EXPD data ----")

        def ids = new TreeSet()
        MergeStatusDAO mergeStatusDAO = toolBox.mergeStatusDAO
        def cache = toolBox.toolBelt.persistenceCache

        ids.addAll(mergeStatusDAO.findUnmergedSets())
        ids.addAll(mergeStatusDAO.findFailedSets())
        ids.addAll(mergeStatusDAO.findSetsWithEditedNav())
        ids.addAll(mergeStatusDAO.findUpdatedSets())

        for (id in ids) {

            if (id != null) {
                try {
                    doMerge(id)
                    cache.clear()
                }
                catch (Exception e) {
                    log.error("An error occurred while trying to merge VideoArchivSet with id = ${id}", e)
                }
            }
        }
        
    }

    /**
     * Merges a {@link VideoArchiveSet} by it's primary-key (id)
     * 
     * @param id<commit_msg>Changed MBARI's data merge to try latest dives first<commit_after>package vars.annotation

import java.text.SimpleDateFormat
import java.util.concurrent.BlockingQueue
import java.util.concurrent.TimeUnit
import java.util.concurrent.LinkedBlockingQueue
import org.slf4j.LoggerFactory
import vars.integration.MergeStatusDAO
import org.mbari.vars.integration.MergeEXPDAnnotations
import vars.integration.MergeFunction
import org.mbari.expd.UberDatum

class DatabaseUtility {

    final log = LoggerFactory.getLogger(DatabaseUtility.class)
    final toolBox = new vars.ToolBox()


    def DatabaseUtility() {
    }

    void merge() {

        log.debug("----- Merging Annotations with EXPD data ----")

        def idSet = new TreeSet()
        MergeStatusDAO mergeStatusDAO = toolBox.mergeStatusDAO
        def cache = toolBox.toolBelt.persistenceCache

        idSet.addAll(mergeStatusDAO.findUnmergedSets())
        idSet.addAll(mergeStatusDAO.findFailedSets())
        idSet.addAll(mergeStatusDAO.findSetsWithEditedNav())
        idSet.addAll(mergeStatusDAO.findUpdatedSets())

        def ids = []
        ids.addAll(idSet)
        ids = ids.sort().reverse(); // Do most recently added first.

        for (id in ids) {

            if (id != null) {
                try {
                    doMerge(id)
                    cache.clear()
                }
                catch (Exception e) {
                    log.error("An error occurred while trying to merge VideoArchivSet with id = ${id}", e)
                }
            }
        }
        
    }

    /**
     * Merges a {@link VideoArchiveSet} by it's primary-key (id)
     * 
     * @param id
{'fraction_non_alphanumeric': 0.09263959390862944, 'fraction_numerical': 0.0006345177664974619, 'mean_word_length': 2.2505154639175258, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3114986', 'n_tokens_mistral': 921, 'n_tokens_neox': 878, 'n_words': 193}