<commit_before>import securehash, os, osproc, algorithm, strutils, times, hashes

# When asset packing algorithm changes, we should increase `hashVersion`
# to invalidate old caches.
const hashVersion = 2

const useExperimentalOptimization = false

proc dirHash*(path: string, profile: string = ""): string =
    var hasSound = false

    when useExperimentalOptimization:
        # Use git
        var lsFilesArgs = ["ls-files", "-o", "--exclude-standard", path]
        let untracked = execProcess("git", lsFilesArgs, options = {poUsePath})
        var gitStageArgs = @["add"]
        for i in untracked.splitLines():
            gitStageArgs.add(i)
        if gitStageArgs.len > 1:
            discard execProcess("git", gitStageArgs, options = {poUsePath})
        lsFilesArgs[1] = "-smd"
        var all = execProcess("git", lsFilesArgs, options = {poUsePath})
        if gitStageArgs.len > 1:
            gitStageArgs[0] = "reset"
            discard execProcess("git", gitStageArgs, options = {poUsePath})
        for ln in splitLines(all):
            if ln.endsWith(".wav"):
                hasSound = true
                break
        if hasSound:
            all &= profile
        result = ($secureHash(all)).toLowerAscii()
    else:
        var allFiles = newSeq[string]()
        for f in walkDirRec(path):
            let sf = f.splitFile()
            if not sf.name.startsWith('.'):
                if sf.ext == ".wav":
                    hasSound = true
                allFiles.add(f)

        allFiles.sort(system.cmp[string])

        var hashStr = ""
        for f in allFiles:
            if path.len > 0:
                hashStr &= f.substr(path.len + 1) & ":"
            else:
                hashStr &= f & ":"
            hashStr &= $hash(readFile(f)) & ";"
        if hasSound:
            hashStr &= profile
        hashStr &= $hashVersion

        result = ($secureHash(hashStr)).toLowerAscii()

proc copyResourcesFromCache*(cache, cacheHash, dst: string) =
    let hashFile = dst / ".hash"
    if fileExists(hashFile) and readFile(hashFile) == cacheHash: return
    removeDir(dst)
    createDir(dst)
    copyDir(cache, dst)
    writeFile(hashFile, cacheHash)

proc getCache*(cacheOverride: string = nil): string =
    if cacheOverride.len > 0: return expandTilde(cacheOverride)
    result = getEnv("ROD_ASSET_CACHE")
    if result.len > 0: return
    result = getTempDir() / "rod_asset_cache"
<commit_msg>fix asset cache (#158)<commit_after>import securehash, os, osproc, algorithm, strutils, times, hashes

# When asset packing algorithm changes, we should increase `hashVersion`
# to invalidate old caches.
const hashVersion = 2

const useExperimentalOptimization = false

proc dirHash*(path: string, profile: string = ""): string =
    var hasSound = false

    when useExperimentalOptimization:
        # Use git
        var lsFilesArgs = ["ls-files", "-o", "--exclude-standard", path]
        let untracked = execProcess("git", lsFilesArgs, options = {poUsePath})
        var gitStageArgs = @["add"]
        for i in untracked.splitLines():
            gitStageArgs.add(i)
        if gitStageArgs.len > 1:
            discard execProcess("git", gitStageArgs, options = {poUsePath})
        lsFilesArgs[1] = "-smd"
        var all = execProcess("git", lsFilesArgs, options = {poUsePath})
        if gitStageArgs.len > 1:
            gitStageArgs[0] = "reset"
            discard execProcess("git", gitStageArgs, options = {poUsePath})
        for ln in splitLines(all):
            if ln.endsWith(".wav") or ln.endsWith(".mp3") or ln.endsWith(".ogg"):
                hasSound = true
                break
        if hasSound:
            all &= profile
        result = ($secureHash(all)).toLowerAscii()
    else:
        var allFiles = newSeq[string]()
        for f in walkDirRec(path):
            let sf = f.splitFile()
            if not sf.name.startsWith('.'):
                if sf.ext == ".wav" or sf.ext == ".mp3" or sf.ext == ".ogg":
                    hasSound = true
                allFiles.add(f)

        allFiles.sort(system.cmp[string])

        var hashStr = ""
        for f in allFiles:
            if path.len > 0:
                hashStr &= f.substr(path.len + 1) & ":"
            else:
                hashStr &= f & ":"
            hashStr &= $hash(readFile(f)) & ";"
        if hasSound:
            hashStr &= profile
        hashStr &= $hashVersion

        result = ($secureHash(hashStr)).toLowerAscii()

proc copyResourcesFromCache*(cache, cacheHash, dst: string) =
    let hashFile = dst / ".hash"
    if fileExists(hashFile) and readFile(hashFile) == cacheHash: return
    removeDir(dst)
    createDir(dst)
    copyDir(cache, dst)
    writeFile(hashFile, cacheHash)

proc getCache*(cacheOverride: string = nil): string =
    if cacheOverride.len > 0: return expandTilde(cacheOverride)
    result = getEnv("ROD_ASSET_CACHE")
    if result.len > 0: return
    result = getTempDir() / "rod_asset_cache"

{'fraction_non_alphanumeric': 0.11924391715262417, 'fraction_numerical': 0.004624974864267042, 'mean_word_length': 2.242503259452412, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6507506', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1444, 'n_words': 387}