<commit_before>package whelk.tools

import groovy.transform.CompileStatic

import java.nio.charset.Charset
import java.nio.file.Files
import java.nio.file.Paths

import org.codehaus.jackson.map.ObjectMapper

import whelk.importer.MySQLLoader

@CompileStatic
class MySQLToMarcJSONDumper {

    static dump(String connectionUrl, String collection, String dumpFileName) {
        def mapper = new ObjectMapper()
        def loader = new MySQLLoader(connectionUrl, collection)
        def dumpWriter = Files.newBufferedWriter(Paths.get(dumpFileName),
                Charset.forName("UTF-8"))

        def counter = 0
        def startTime = System.currentTimeMillis()

        try {
            loader.run { doc, specs ->

                dumpWriter.writeLine(mapper.writeValueAsString(doc))

                if (++counter % 1000 == 0) {
                    def elapsedSecs = (System.currentTimeMillis() - startTime) / 1000
                    if (elapsedSecs > 0) {
                        def docsPerSec = counter / elapsedSecs
                        println "Working. Currently $counter documents saved. Crunching $docsPerSec docs / s"
                    }
                }
            }
        } finally {
            dumpWriter.close()
        }

        def endSecs = (System.currentTimeMillis() - startTime) / 1000
        println "Done. Processed $counter documents in $endSecs seconds."
    }

}<commit_msg>Output to stdout if no file given to MySQLToMarcJSONDumper<commit_after>package whelk.tools

import groovy.transform.CompileStatic

import java.nio.charset.Charset
import java.nio.file.Files
import java.nio.file.Paths

import org.codehaus.jackson.map.ObjectMapper

import whelk.importer.MySQLLoader

@CompileStatic
class MySQLToMarcJSONDumper {

    static dump(String connectionUrl, String collection, String dumpFileName) {
        def mapper = new ObjectMapper()
        def loader = new MySQLLoader(connectionUrl, collection)
        def dumpWriter = dumpFileName ?
                Files.newBufferedWriter(Paths.get(dumpFileName), Charset.forName("UTF-8"))
                : new BufferedWriter(System.out.newWriter())

        def counter = 0
        def startTime = System.currentTimeMillis()

        try {
            loader.run { doc, specs ->

                dumpWriter.writeLine(mapper.writeValueAsString(doc))

                if (++counter % 1000 == 0) {
                    def elapsedSecs = (System.currentTimeMillis() - startTime) / 1000
                    if (elapsedSecs > 0) {
                        def docsPerSec = counter / elapsedSecs
                        System.err.println "Working. Currently $counter documents saved. Crunching $docsPerSec docs / s"
                    }
                }
            }
        } finally {
            dumpWriter.close()
        }

        def endSecs = (System.currentTimeMillis() - startTime) / 1000
        System.err.println "Done. Processed $counter documents in $endSecs seconds."
    }

}
{'fraction_non_alphanumeric': 0.0846489754786698, 'fraction_numerical': 0.010749076251259657, 'mean_word_length': 2.0637860082304527, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2404459', 'n_tokens_mistral': 862, 'n_tokens_neox': 780, 'n_words': 189}