<commit_before>import sys, os, random, errno, stat
import vars, jwack, state
from helpers import log, log_, debug2, err, unlink, close_on_exec


def _possible_do_files(t):
    yield "%s.do" % t, t, ''
    dirname,filename = os.path.split(t)
    l = filename.split('.')
    l[0] = os.path.join(dirname, l[0])
    for i in range(1,len(l)+1):
        basename = '.'.join(l[:i])
        ext = '.'.join(l[i:])
        if ext: ext = '.' + ext
        yield (os.path.join(dirname, "default%s.do" % ext),
               os.path.join(dirname, basename), ext)


def _find_do_file(t):
    for dofile,basename,ext in _possible_do_files(t):
        debug2('%s: %s ?\n' % (t, dofile))
        if os.path.exists(dofile):
            state.File(name=t).add_dep('m', dofile)
            return dofile,basename,ext
        else:
            state.File(name=t).add_dep('c', dofile)
    return None,None,None


def _nice(t):
    return state.relpath(t, vars.STARTDIR)


def _try_stat(filename):
    try:
        return os.stat(filename)
    except OSError, e:
        if e.errno == errno.ENOENT:
            return None
        else:
            raise


class BuildJob:
    def __init__(self, t, lock, shouldbuildfunc, donefunc):
        self.t = t
        self.tmpname = '%s.redo.tmp' % t
        self.lock = lock
        self.shouldbuildfunc = shouldbuildfunc
        self.donefunc = donefunc
        self.before_t = _try_stat(self.t)

    def start(self):
        assert(self.lock.owned)
        t = self.t
        f = state.File(name=t)
        tmpname = self.tmpname
        if not self.shouldbuildfunc(t):
            # target doesn't need to be built; skip the whole task
            return self._after2(0)
        if (os.path.exists(t) and not os.path.exists(t + '/.')
             and not f.is_generated):
            # an existing source file that was not generated by us.
            # This step is mentioned by djb in his notes.
            # For example, a rule called default.c.do could be used to try
            # to produce hello.c, but we don't want that to happen if
            # hello.c was created by the end user.
            # FIXME: always refuse to redo any file that was modified outside
            # of redo?  That would make it easy for someone to override a
            # file temporarily, and could be undone by deleting the file.
            debug2("-- static (%r)\n" % t)
            f.set_static()
            f.save()
            return self._after2(0)
        f.zap_deps()
        (dofile, basename, ext) = _find_do_file(t)
        if not dofile:
            if os.path.exists(t):
                f.is_generated = False
                f.set_static()
                f.save()
                return self._after2(0)
            else:
                err('no rule to make %r\n' % t)
                return self._after2(1)
        unlink(tmpname)
        ffd = os.open(tmpname, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
        close_on_exec(ffd, True)
        self.f = os.fdopen(ffd, 'w+')
        # this will run in the dofile's directory, so use only basenames here
        argv = ['sh', '-e',
                os.path.basename(dofile),
                os.path.basename(basename),  # target name (extension removed)
                ext,  # extension (if any), including leading dot
                os.path.basename(tmpname)  # randomized output file name
                ]
        if vars.VERBOSE: argv[1] += 'v'
        if vars.XTRACE: argv[1] += 'x'
        if vars.VERBOSE or vars.XTRACE: log_('\n')
        log('%s\n' % _nice(t))
        self.argv = argv
        f.is_generated = True
        f.save()
        dof = state.File(name=dofile)
        dof.set_static()
        dof.save()
        state.commit()
        jwack.start_job(t, self._do_subproc, self._after)

    def _do_subproc(self):
        # careful: REDO_PWD was the PWD relative to the STARTPATH at the time
        # we *started* building the current target; but that target ran
        # redo-ifchange, and it might have done it from a different directory
        # than we started it in.  So os.getcwd() might be != REDO_PWD right
        # now.
        dn = os.path.dirname(self.t)
        newp = os.path.realpath(dn)
        os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR)
        os.environ['REDO_TARGET'] = os.path.basename(self.t)
        os.environ['REDO_DEPTH'] = vars.DEPTH + '  '
        if dn:
            os.chdir(dn)
        os.dup2(self.f.fileno(), 1)
        os.close(self.f.fileno())
        close_on_exec(1, False)
        if vars.VERBOSE or vars.XTRACE: log_('* %s\n' % ' '.join(self.argv))
        os.execvp(self.argv[0], self.argv)
        assert(0)
        # returns only if there's an exception

    def _after(self, t, rv):
        try:
            state.check_sane()
            rv = self._after1(t, rv)
        finally:
            self._after2(rv)

    def _after1(self, t, rv):
        f = self.f
        tmpname = self.tmpname
        before_t = self.before_t
        after_t = _try_stat(t)
        before_tmp = os.fstat(f.fileno())
        after_tmp = _try_stat(tmpname)
        after_where = os.lseek(f.fileno(), 0, os.SEEK_CUR)
        if after_t != before_t and not stat.S_ISDIR(after_t.st_mode):
            err('%r modified %r directly!\n' % (self.argv[2], t))
            err('...you should update $3 (a temp file) instead of $1.\n')
            rv = 206
        elif after_tmp and before_tmp != after_tmp and before_tmp.st_size > 0:
            err('%r wrote to stdout *and* replaced $3.\n' % self.argv[2])
            err('...you should write status messages to stderr, not stdout.\n')
            rv = 207
        elif after_where > 0 and after_tmp and after_tmp.st_size != after_where:
            err('%r wrote differing data to stdout and $3.\n' % self.argv[2])
            err('...you should write status messages to stderr, not stdout.\n')
            rv = 208
        if rv==0:
            if os.path.exists(tmpname) and os.stat(tmpname).st_size:
                # there's a race condition here, but if the tmpfile disappears
                # at *this* point you deserve to get an error, because you're
                # doing something totally scary.
                os.rename(tmpname, t)
            else:
                unlink(tmpname)
            sf = state.File(name=t)
            sf.is_generated=True
            sf.update_stamp()
            sf.set_changed()
            sf.save()
        else:
            unlink(tmpname)
            sf = state.File(name=t)
            sf.stamp = None
            sf.set_changed()
            sf.save()
        f.close()
        if rv != 0:
            err('%s: exit code %d\n' % (_nice(t),rv))
        else:
            if vars.VERBOSE or vars.XTRACE:
                log('%s (done)\n\n' % _nice(t))
        return rv

    def _after2(self, rv):
        try:
            self.donefunc(self.t, rv)
            assert(self.lock.owned)
            state.commit()
        finally:
            self.lock.unlock()


def main(targets, shouldbuildfunc):
    retcode = [0]  # a list so that it can be reassigned from done()
    if vars.SHUFFLE:
        random.shuffle(targets)

    locked = []

    def done(t, rv):
        if rv:
            retcode[0] = 1

    for i in range(len(targets)):
        t = targets[i]<commit_msg>builder.py: don't import the 'random' module unless we need it.<commit_after>import sys, os, errno, stat
import vars, jwack, state
from helpers import log, log_, debug2, err, unlink, close_on_exec


def _possible_do_files(t):
    yield "%s.do" % t, t, ''
    dirname,filename = os.path.split(t)
    l = filename.split('.')
    l[0] = os.path.join(dirname, l[0])
    for i in range(1,len(l)+1):
        basename = '.'.join(l[:i])
        ext = '.'.join(l[i:])
        if ext: ext = '.' + ext
        yield (os.path.join(dirname, "default%s.do" % ext),
               os.path.join(dirname, basename), ext)


def _find_do_file(t):
    for dofile,basename,ext in _possible_do_files(t):
        debug2('%s: %s ?\n' % (t, dofile))
        if os.path.exists(dofile):
            state.File(name=t).add_dep('m', dofile)
            return dofile,basename,ext
        else:
            state.File(name=t).add_dep('c', dofile)
    return None,None,None


def _nice(t):
    return state.relpath(t, vars.STARTDIR)


def _try_stat(filename):
    try:
        return os.stat(filename)
    except OSError, e:
        if e.errno == errno.ENOENT:
            return None
        else:
            raise


class BuildJob:
    def __init__(self, t, lock, shouldbuildfunc, donefunc):
        self.t = t
        self.tmpname = '%s.redo.tmp' % t
        self.lock = lock
        self.shouldbuildfunc = shouldbuildfunc
        self.donefunc = donefunc
        self.before_t = _try_stat(self.t)

    def start(self):
        assert(self.lock.owned)
        t = self.t
        f = state.File(name=t)
        tmpname = self.tmpname
        if not self.shouldbuildfunc(t):
            # target doesn't need to be built; skip the whole task
            return self._after2(0)
        if (os.path.exists(t) and not os.path.exists(t + '/.')
             and not f.is_generated):
            # an existing source file that was not generated by us.
            # This step is mentioned by djb in his notes.
            # For example, a rule called default.c.do could be used to try
            # to produce hello.c, but we don't want that to happen if
            # hello.c was created by the end user.
            # FIXME: always refuse to redo any file that was modified outside
            # of redo?  That would make it easy for someone to override a
            # file temporarily, and could be undone by deleting the file.
            debug2("-- static (%r)\n" % t)
            f.set_static()
            f.save()
            return self._after2(0)
        f.zap_deps()
        (dofile, basename, ext) = _find_do_file(t)
        if not dofile:
            if os.path.exists(t):
                f.is_generated = False
                f.set_static()
                f.save()
                return self._after2(0)
            else:
                err('no rule to make %r\n' % t)
                return self._after2(1)
        unlink(tmpname)
        ffd = os.open(tmpname, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
        close_on_exec(ffd, True)
        self.f = os.fdopen(ffd, 'w+')
        # this will run in the dofile's directory, so use only basenames here
        argv = ['sh', '-e',
                os.path.basename(dofile),
                os.path.basename(basename),  # target name (extension removed)
                ext,  # extension (if any), including leading dot
                os.path.basename(tmpname)  # randomized output file name
                ]
        if vars.VERBOSE: argv[1] += 'v'
        if vars.XTRACE: argv[1] += 'x'
        if vars.VERBOSE or vars.XTRACE: log_('\n')
        log('%s\n' % _nice(t))
        self.argv = argv
        f.is_generated = True
        f.save()
        dof = state.File(name=dofile)
        dof.set_static()
        dof.save()
        state.commit()
        jwack.start_job(t, self._do_subproc, self._after)

    def _do_subproc(self):
        # careful: REDO_PWD was the PWD relative to the STARTPATH at the time
        # we *started* building the current target; but that target ran
        # redo-ifchange, and it might have done it from a different directory
        # than we started it in.  So os.getcwd() might be != REDO_PWD right
        # now.
        dn = os.path.dirname(self.t)
        newp = os.path.realpath(dn)
        os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR)
        os.environ['REDO_TARGET'] = os.path.basename(self.t)
        os.environ['REDO_DEPTH'] = vars.DEPTH + '  '
        if dn:
            os.chdir(dn)
        os.dup2(self.f.fileno(), 1)
        os.close(self.f.fileno())
        close_on_exec(1, False)
        if vars.VERBOSE or vars.XTRACE: log_('* %s\n' % ' '.join(self.argv))
        os.execvp(self.argv[0], self.argv)
        assert(0)
        # returns only if there's an exception

    def _after(self, t, rv):
        try:
            state.check_sane()
            rv = self._after1(t, rv)
        finally:
            self._after2(rv)

    def _after1(self, t, rv):
        f = self.f
        tmpname = self.tmpname
        before_t = self.before_t
        after_t = _try_stat(t)
        before_tmp = os.fstat(f.fileno())
        after_tmp = _try_stat(tmpname)
        after_where = os.lseek(f.fileno(), 0, os.SEEK_CUR)
        if after_t != before_t and not stat.S_ISDIR(after_t.st_mode):
            err('%r modified %r directly!\n' % (self.argv[2], t))
            err('...you should update $3 (a temp file) instead of $1.\n')
            rv = 206
        elif after_tmp and before_tmp != after_tmp and before_tmp.st_size > 0:
            err('%r wrote to stdout *and* replaced $3.\n' % self.argv[2])
            err('...you should write status messages to stderr, not stdout.\n')
            rv = 207
        elif after_where > 0 and after_tmp and after_tmp.st_size != after_where:
            err('%r wrote differing data to stdout and $3.\n' % self.argv[2])
            err('...you should write status messages to stderr, not stdout.\n')
            rv = 208
        if rv==0:
            if os.path.exists(tmpname) and os.stat(tmpname).st_size:
                # there's a race condition here, but if the tmpfile disappears
                # at *this* point you deserve to get an error, because you're
                # doing something totally scary.
                os.rename(tmpname, t)
            else:
                unlink(tmpname)
            sf = state.File(name=t)
            sf.is_generated=True
            sf.update_stamp()
            sf.set_changed()
            sf.save()
        else:
            unlink(tmpname)
            sf = state.File(name=t)
            sf.stamp = None
            sf.set_changed()
            sf.save()
        f.close()
        if rv != 0:
            err('%s: exit code %d\n' % (_nice(t),rv))
        else:
            if vars.VERBOSE or vars.XTRACE:
                log('%s (done)\n\n' % _nice(t))
        return rv

    def _after2(self, rv):
        try:
            self.donefunc(self.t, rv)
            assert(self.lock.owned)
            state.commit()
        finally:
            self.lock.unlock()


def main(targets, shouldbuildfunc):
    retcode = [0]  # a list so that it can be reassigned from done()
    if vars.SHUFFLE:
        import random
        random.shuffle(targets)

    locked = []

    def done(t, rv):
        if rv:
            retcode[0] = 1

    for i in range(len(targets)):
        t = targets[i]
{'fraction_non_alphanumeric': 0.11802809218600846, 'fraction_numerical': 0.007363971089594982, 'mean_word_length': 1.7471436598613972, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6087029', 'n_tokens_mistral': 4862, 'n_tokens_neox': 4726, 'n_words': 1275}