<commit_before>#!/usr/bin/python
import sys, os, errno
import vars, state, builder
from helpers import debug, err, mkdirp, unlink


def _dirty_deps(t, depth):
    debug('%s?%s\n' % (depth, t))
    if state.isbuilt(t):
        debug('%s-- DIRTY (built)\n' % depth)
        return True  # has already been built during this session
    if state.ismarked(t):
        debug('%s-- CLEAN (marked)\n' % depth)
        return False  # has already been checked during this session
    
    stamptime = state.stamped(t)
    if stamptime == None:
        debug('%s-- DIRTY (no stamp)\n' % depth)
        return True

    try:
        realtime = os.stat(t).st_mtime
    except OSError:
        realtime = 0

    if stamptime != realtime:
        debug('%s-- DIRTY (mtime)\n' % depth)
        return True
    
    for mode,name in state.deps(t):
        if mode == 'c':
            if os.path.exists(name):
                debug('%s-- DIRTY (created)\n' % depth)
                return True
        elif mode == 'm':
            if dirty_deps(os.path.join(vars.BASE, name), depth + '  '):
                debug('%s-- DIRTY (sub)\n' % depth)
                return True
    state.mark(t)
    return False


def dirty_deps(t, depth):
    if _dirty_deps(t, depth):
        state.unstamp(t)
        return True
    return False


def should_build(t):
    return not state.isbuilt(t) and dirty_deps(t, depth = '')


def maybe_build(t):
    if should_build(t):<commit_msg>redo-ifchange: unstamp() was in the wrong spot, causing unnecessary rebuilds.<commit_after>#!/usr/bin/python
import sys, os, errno
import vars, state, builder
from helpers import debug, err, mkdirp, unlink


def dirty_deps(t, depth):
    debug('%s?%s\n' % (depth, t))
    if state.isbuilt(t):
        debug('%s-- DIRTY (built)\n' % depth)
        return True  # has already been built during this session
    if state.ismarked(t):
        debug('%s-- CLEAN (marked)\n' % depth)
        return False  # has already been checked during this session
    
    stamptime = state.stamped(t)
    if stamptime == None:
        debug('%s-- DIRTY (no stamp)\n' % depth)
        return True

    try:
        realtime = os.stat(t).st_mtime
    except OSError:
        realtime = 0

    if stamptime != realtime:
        debug('%s-- DIRTY (mtime)\n' % depth)
        return True
    
    for mode,name in state.deps(t):
        if mode == 'c':
            if os.path.exists(name):
                debug('%s-- DIRTY (created)\n' % depth)
                return True
        elif mode == 'm':
            if dirty_deps(os.path.join(vars.BASE, name), depth + '  '):
                debug('%s-- DIRTY (sub)\n' % depth)
                state.unstamp(t)  # optimization for future callers
                return True
    state.mark(t)
    return False


def should_build(t):
    return not state.isbuilt(t) and dirty_deps(t, depth = '')


def maybe_build(t):
    if should_build(t):
{'fraction_non_alphanumeric': 0.12992800822763112, 'fraction_numerical': 0.0006856359273225917, 'mean_word_length': 2.0911016949152543, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5348345', 'n_tokens_mistral': 994, 'n_tokens_neox': 964, 'n_words': 270}