<commit_before>import securehash, os, osproc, algorithm, strutils, times, hashes

import settings

# When asset packing algorithm changes, we should increase `hashVersion`
# to invalidate old caches.
const hashVersion = 3

const audioFileExtensions = [".wav", ".ogg", "mp3"]

const max_copy_attempts = 5

proc isAudio(path: string): bool {.inline.} =
    for e in audioFileExtensions:
        if path.endsWith(e): return true

proc isGraphics(path: string): bool {.inline.} = path.endsWith(".png")

proc gitDirHash(path: string): string =
    if getEnv("ROD_HASH_METHOD") != "git":
        return nil

    let (outp, errC) = execCmdEx("git ls-tree -d HEAD " & path)
    if errC == 0:
        let comps = outp.split()
        if comps.len > 3:
            return comps[2]

proc isHiddenFile(path: string): bool =
    let slash = path.rfind({'/', '\\'})
    if slash != -1:
        result = path[slash + 1] == '.'

proc dirHashImplGit(path, baseHash: string, s: Settings): string {.inline.} =
    result = newStringOfCap(2048)
    result &= baseHash
    result &= ';'

    let (outp, errC) = execCmdEx("git status -s " & path)
    if errC != 0:
        raise newException(Exception, "git status returned " & $errC)
    # echo "OUTP: ", outp
    for ln in outp.splitLines:
        if ln.len == 0: continue
        let f = ln[3 .. ^1]
        if f.isHiddenFile(): continue

        result &= f
        result &= ';'
        if ln[0] != 'D':
            let modTime = getLastModificationTime(f)
            result &= $modTime
            result &= ';'

    var hasSound = false
    var hasGraphics = false
    for f in walkDirRec(path):
        if not f.isHiddenFile():
            if not hasSound and f.isAudio():
                hasSound = true
                if hasGraphics: break
            elif not hasGraphics and f.isGraphics():
                hasGraphics = true
                if hasSound: break<commit_msg>Some fixes to git status check<commit_after>import securehash, os, osproc, algorithm, strutils, times, hashes

import settings

# When asset packing algorithm changes, we should increase `hashVersion`
# to invalidate old caches.
const hashVersion = 3

const audioFileExtensions = [".wav", ".ogg", "mp3"]

const max_copy_attempts = 5

proc isAudio(path: string): bool {.inline.} =
    for e in audioFileExtensions:
        if path.endsWith(e): return true

proc isGraphics(path: string): bool {.inline.} = path.endsWith(".png")

proc gitDirHash(path: string): string =
    if getEnv("ROD_HASH_METHOD") != "git":
        return nil

    let (outp, errC) = execCmdEx("git ls-tree -d HEAD " & path)
    if errC == 0:
        let comps = outp.split()
        if comps.len > 3:
            return comps[2]

proc isHiddenFile(path: string): bool =
    let slash = path.rfind({'/', '\\'})
    if slash != -1:
        result = path[slash + 1] == '.'

proc gitStatus(statusLine: string, status: char): bool {.inline.} =
    statusLine[0] == status or statusLine[1] == status

proc getFileNameFromMovedGitStatusLine(statusLine: string): string {.inline.} =
    const separator = " -> "
    let idx = statusLine.rfind(separator)
    result = statusLine[idx + separator.len .. ^1]

proc dirHashImplGit(path, baseHash: string, s: Settings): string {.inline.} =
    result = newStringOfCap(2048)
    result &= baseHash
    result &= ';'

    let (outp, errC) = execCmdEx("git status -s " & path)
    if errC != 0:
        raise newException(Exception, "git status returned " & $errC)

    for ln in outp.splitLines:
        if ln.len == 0: continue
        var f = ln[3 .. ^1]
        if f.isHiddenFile(): continue

        result &= f
        result &= ';'

        if not ln.gitStatus('D'):
            if ln.gitStatus('R'):
                f = getFileNameFromMovedGitStatusLine(ln)

            let modTime = getLastModificationTime(f)
            result &= $modTime
            result &= ';'

    var hasSound = false
    var hasGraphics = false
    for f in walkDirRec(path):
        if not f.isHiddenFile():
            if not hasSound and f.isAudio():
                hasSound = true
                if hasGraphics: break
            elif not hasGraphics and f.isGraphics():
                hasGraphics = true
                if hasSound: break
{'fraction_non_alphanumeric': 0.1228235294117647, 'fraction_numerical': 0.008470588235294117, 'mean_word_length': 2.3551696921862666, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1864750', 'n_tokens_mistral': 1361, 'n_tokens_neox': 1293, 'n_words': 403}