<commit_before>import os, strutils, times, osproc, sets, logging
import imgtool, asset_cache, migrator
import settings except hash
import json except hash
import tempfile


template settingsWithCmdLine(): Settings =
    let s = newSettings()
    s.audio.extension = audio
    s.graphics.downsampleRatio = downsampleRatio
    s.graphics.compressOutput = not nocompress
    s.graphics.extrusion = extrusion
    s.graphics.disablePotAdjustment = disablePotAdjustment
    s.graphics.packCompositions = packCompositions
    s.graphics.compressToPVR = compressToPVR
    s.graphics.quantizeExceptions = exceptions & "," & noquant
    s.graphics.posterizeExceptions = noposterize
    s

proc hash(audio: string = "ogg", downsampleRatio: float = 1.0, nocompress: bool = false,
    compressToPVR: bool = false, extrusion: int = 1, disablePotAdjustment: bool = false,
    packCompositions: bool = false,
    exceptions: string = "", noposterize: string = "", noquant: string = "",
    path: string) =
    let s = settingsWithCmdLine()
    echo dirHash(path, s)

var gAudioConvTool = ""

proc audioConvTool(): string =
    if gAudioConvTool.len == 0:
        gAudioConvTool = findExe("ffmpeg")
        if gAudioConvTool.len == 0:
            gAudioConvTool = findExe("avconv")
    result = gAudioConvTool

let compressAudio = false

proc convertAudio(fromFile, toFile: string, mp3: bool) =
    var args = @["-i", fromFile, "-y", "-loglevel", "warning"]
    if mp3:
        args.add(["-acodec", "libmp3lame", "-write_xing", "0"])
    else: # ogg
        args.add(["-acodec", "libvorbis"])

    if compressAudio:
        let numChannels = 1
        let sampleRate = 11025
        args.add(["-ac", $numChannels, "-ar", $sampleRate])

    args.add(toFile)
    echo audioConvTool().execProcess(args, options={poStdErrToStdOut})

proc copyRemainingAssets(tool: ImgTool, src, dst, audioFmt: string, copiedFiles: var seq[string]) =
    let isMp3 = audioFmt == "mp3"
    for r in walkDirRec(src):
        let sf = r.splitFile()
        if not sf.name.startsWith('.'):
            var reldst = substr(r, src.len + 1)
            let d = dst / reldst
            var doCopy = false
            var doIndex = false
            case sf.ext
            of ".png":
                if unixToNativePath(r) notin tool.processedImages:
                    doCopy = true
            of ".wav", ".mp3", ".ogg":
                createDir(d.parentDir())
                let dest = d.changeFileExt(audioFmt)
                reldst = reldst.changeFileExt(audioFmt)
                doIndex = true
                echo "Converting/compressing audio ", r
                if isMp3:
                    convertAudio(r, dest, true)
                else:
                    convertAudio(r, dest, false)
            of ".json":
                doIndex = not tool.packCompositions
            of ".rab":
                discard
            else:
                doCopy = true

            if doCopy or doIndex:
                copiedFiles.add(reldst.replace('\\', '/'))

                if doCopy:
                    echo "Copying asset: ", r
                    createDir(d.parentDir())
                    copyFile(r, d)

proc pack(cache: string = "", exceptions: string = "", noposterize: string = "", noquant: string = "", compressToPVR: bool = false, nocompress: bool = false,
        downsampleRatio: float = 1.0, extrusion: int = 1, createIndex: bool = false,
        disablePotAdjustment: bool = false, audio: string = "ogg", packCompositions: bool = false,
        onlyCache: bool = false,
        src, dst: string) =
    addHandler(newConsoleLogger())
    let src = expandTilde(src)
    let dst = expandTilde(dst)
    let cache = getCache(cache)
    let s = settingsWithCmdLine()
    let h = dirHash(src, s)
    createDir(cache)
    let c = cache / h
    echo "rodasset Cache: ", c
    if not dirExists(c):
        let tmpCacheDir = mkdtemp(h, "_tmp")
        var tool = newImgTool()

        tool.noquant = @[]
        tool.noposterize = @[]
        for f in walkDirRec(src):
            if f.endsWith(".json"):
                tool.compositionPaths.add(f)
        for e in split(exceptions, ","): tool.noquant.add(e)
        for e in split(noquant, ","): tool.noquant.add(e)
        for e in split(noposterize, ","): tool.noposterize.add(e)
        tool.originalResPath = src<commit_msg>Remove logger. Nimx logger is still there.<commit_after>import os, strutils, times, osproc, sets
import imgtool, asset_cache, migrator
import settings except hash
import json except hash
import tempfile


template settingsWithCmdLine(): Settings =
    let s = newSettings()
    s.audio.extension = audio
    s.graphics.downsampleRatio = downsampleRatio
    s.graphics.compressOutput = not nocompress
    s.graphics.extrusion = extrusion
    s.graphics.disablePotAdjustment = disablePotAdjustment
    s.graphics.packCompositions = packCompositions
    s.graphics.compressToPVR = compressToPVR
    s.graphics.quantizeExceptions = exceptions & "," & noquant
    s.graphics.posterizeExceptions = noposterize
    s

proc hash(audio: string = "ogg", downsampleRatio: float = 1.0, nocompress: bool = false,
    compressToPVR: bool = false, extrusion: int = 1, disablePotAdjustment: bool = false,
    packCompositions: bool = false,
    exceptions: string = "", noposterize: string = "", noquant: string = "",
    path: string) =
    let s = settingsWithCmdLine()
    echo dirHash(path, s)

var gAudioConvTool = ""

proc audioConvTool(): string =
    if gAudioConvTool.len == 0:
        gAudioConvTool = findExe("ffmpeg")
        if gAudioConvTool.len == 0:
            gAudioConvTool = findExe("avconv")
    result = gAudioConvTool

let compressAudio = false

proc convertAudio(fromFile, toFile: string, mp3: bool) =
    var args = @["-i", fromFile, "-y", "-loglevel", "warning"]
    if mp3:
        args.add(["-acodec", "libmp3lame", "-write_xing", "0"])
    else: # ogg
        args.add(["-acodec", "libvorbis"])

    if compressAudio:
        let numChannels = 1
        let sampleRate = 11025
        args.add(["-ac", $numChannels, "-ar", $sampleRate])

    args.add(toFile)
    echo audioConvTool().execProcess(args, options={poStdErrToStdOut})

proc copyRemainingAssets(tool: ImgTool, src, dst, audioFmt: string, copiedFiles: var seq[string]) =
    let isMp3 = audioFmt == "mp3"
    for r in walkDirRec(src):
        let sf = r.splitFile()
        if not sf.name.startsWith('.'):
            var reldst = substr(r, src.len + 1)
            let d = dst / reldst
            var doCopy = false
            var doIndex = false
            case sf.ext
            of ".png":
                if unixToNativePath(r) notin tool.processedImages:
                    doCopy = true
            of ".wav", ".mp3", ".ogg":
                createDir(d.parentDir())
                let dest = d.changeFileExt(audioFmt)
                reldst = reldst.changeFileExt(audioFmt)
                doIndex = true
                echo "Converting/compressing audio ", r
                if isMp3:
                    convertAudio(r, dest, true)
                else:
                    convertAudio(r, dest, false)
            of ".json":
                doIndex = not tool.packCompositions
            of ".rab":
                discard
            else:
                doCopy = true

            if doCopy or doIndex:
                copiedFiles.add(reldst.replace('\\', '/'))

                if doCopy:
                    echo "Copying asset: ", r
                    createDir(d.parentDir())
                    copyFile(r, d)

proc pack(cache: string = "", exceptions: string = "", noposterize: string = "", noquant: string = "", compressToPVR: bool = false, nocompress: bool = false,
        downsampleRatio: float = 1.0, extrusion: int = 1, createIndex: bool = false,
        disablePotAdjustment: bool = false, audio: string = "ogg", packCompositions: bool = false,
        onlyCache: bool = false,
        src, dst: string) =
    let src = expandTilde(src)
    let dst = expandTilde(dst)
    let cache = getCache(cache)
    let s = settingsWithCmdLine()
    let h = dirHash(src, s)
    createDir(cache)
    let c = cache / h
    echo "rodasset Cache: ", c
    if not dirExists(c):
        let tmpCacheDir = mkdtemp(h, "_tmp")
        var tool = newImgTool()

        tool.noquant = @[]
        tool.noposterize = @[]
        for f in walkDirRec(src):
            if f.endsWith(".json"):
                tool.compositionPaths.add(f)
        for e in split(exceptions, ","): tool.noquant.add(e)
        for e in split(noquant, ","): tool.noquant.add(e)
        for e in split(noposterize, ","): tool.noposterize.add(e)
        tool.originalResPath = src
{'fraction_non_alphanumeric': 0.11466359976998275, 'fraction_numerical': 0.005290396779758482, 'mean_word_length': 2.212412264499446, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7297064', 'n_tokens_mistral': 2699, 'n_tokens_neox': 2591, 'n_words': 689}