<commit_before>import os, stat, struct, tempfile
from bup.helpers import *

EMPTY_SHA = '\0'*20
FAKE_SHA = '\x01'*20
INDEX_HDR = 'BUPI\0\0\0\2'
INDEX_SIG = '!IIIIIQII20sHII'
ENTLEN = struct.calcsize(INDEX_SIG)
FOOTER_SIG = '!Q'
FOOTLEN = struct.calcsize(FOOTER_SIG)

IX_EXISTS = 0x8000        # file exists on filesystem
IX_HASHVALID = 0x4000     # the stored sha1 matches the filesystem
IX_SHAMISSING = 0x2000    # the stored sha1 object doesn't seem to exist

class Error(Exception):
    pass


class Level:
    def __init__(self, ename, parent):
        self.parent = parent
        self.ename = ename
        self.list = []
        self.count = 0

    def write(self, f):
        (ofs,n) = (f.tell(), len(self.list))
        if self.list:
            count = len(self.list)
            #log('popping %r with %d entries\n' 
            #    % (''.join(self.ename), count))
            for e in self.list:
                e.write(f)
            if self.parent:
                self.parent.count += count + self.count
        return (ofs,n)


def _golevel(level, f, ename, newentry):
    # close nodes back up the tree
    assert(level)
    while ename[:len(level.ename)] != level.ename:
        n = BlankNewEntry(level.ename[-1])
        n.flags |= IX_EXISTS
        (n.children_ofs,n.children_n) = level.write(f)
        level.parent.list.append(n)
        level = level.parent

    # create nodes down the tree
    while len(level.ename) < len(ename):
        level = Level(ename[:len(level.ename)+1], level)

    # are we in precisely the right place?
    assert(ename == level.ename)
    n = newentry or BlankNewEntry(ename and level.ename[-1] or None)
    (n.children_ofs,n.children_n) = level.write(f)
    if level.parent:
        level.parent.list.append(n)
    level = level.parent

    return level


class Entry:
    def __init__(self, basename, name):
        self.basename = str(basename)
        self.name = str(name)
        self.children_ofs = 0
        self.children_n = 0

    def __repr__(self):
        return ("(%s,0x%04x,%d,%d,%d,%d,%d,%s/%s,0x%04x,0x%08x/%d)" 
                % (self.name, self.dev,
                   self.ctime, self.mtime, self.uid, self.gid,
                   self.size, oct(self.mode), oct(self.gitmode),
                   self.flags, self.children_ofs, self.children_n))

    def packed(self):
        try:
            return struct.pack(INDEX_SIG,
                           self.dev, self.ctime, self.mtime, 
                           self.uid, self.gid, self.size, self.mode,
                           self.gitmode, self.sha, self.flags,
                           self.children_ofs, self.children_n)
        except DeprecationWarning, e:
            log('pack error: %s (%r)\n' % (e, self))
            raise

    def from_stat(self, st, tstart):
        old = (self.dev, self.ctime, self.mtime,
               self.uid, self.gid, self.size, self.flags & IX_EXISTS)
        new = (st.st_dev, int(st.st_ctime), int(st.st_mtime),
               st.st_uid, st.st_gid, st.st_size, IX_EXISTS)
        self.dev = st.st_dev
        self.ctime = int(st.st_ctime)
        self.mtime = int(st.st_mtime)
        self.uid = st.st_uid
        self.gid = st.st_gid
        self.size = st.st_size
        self.mode = st.st_mode
        self.flags |= IX_EXISTS
        if int(st.st_ctime) >= tstart or old != new \
              or self.sha == EMPTY_SHA or not self.gitmode:
            self.invalidate()
        self._fixup()
        
    def _fixup(self):
        if self.uid < 0:
            self.uid += 0x100000000
        if self.gid < 0:
            self.gid += 0x100000000
        assert(self.uid >= 0)
        assert(self.gid >= 0)

    def is_valid(self):
        f = IX_HASHVALID|IX_EXISTS
        return (self.flags & f) == f

    def invalidate(self):
        self.flags &= ~IX_HASHVALID

    def validate(self, gitmode, sha):
        assert(sha)
        assert(gitmode)
        self.gitmode = gitmode
        self.sha = sha
        self.flags |= IX_HASHVALID|IX_EXISTS

    def exists(self):
        return not self.is_deleted()

    def sha_missing(self):
        return (self.flags & IX_SHAMISSING) or not (self.flags & IX_HASHVALID)<commit_msg>Fixup ctime/mtime that are outside a 32-bit range.<commit_after>import os, stat, struct, tempfile
from bup.helpers import *

EMPTY_SHA = '\0'*20
FAKE_SHA = '\x01'*20
INDEX_HDR = 'BUPI\0\0\0\2'

# FIXME: guess I should have used 64-bit integers to store the mtime/ctime.
# NTFS mtime=0 corresponds to the year 1600, which can't be stored in a 32-bit
# time_t.  Next time we update the bupindex format, keep that in mind.
INDEX_SIG = '!IIIIIQII20sHII'

ENTLEN = struct.calcsize(INDEX_SIG)
FOOTER_SIG = '!Q'
FOOTLEN = struct.calcsize(FOOTER_SIG)

IX_EXISTS = 0x8000        # file exists on filesystem
IX_HASHVALID = 0x4000     # the stored sha1 matches the filesystem
IX_SHAMISSING = 0x2000    # the stored sha1 object doesn't seem to exist

class Error(Exception):
    pass


class Level:
    def __init__(self, ename, parent):
        self.parent = parent
        self.ename = ename
        self.list = []
        self.count = 0

    def write(self, f):
        (ofs,n) = (f.tell(), len(self.list))
        if self.list:
            count = len(self.list)
            #log('popping %r with %d entries\n' 
            #    % (''.join(self.ename), count))
            for e in self.list:
                e.write(f)
            if self.parent:
                self.parent.count += count + self.count
        return (ofs,n)


def _golevel(level, f, ename, newentry):
    # close nodes back up the tree
    assert(level)
    while ename[:len(level.ename)] != level.ename:
        n = BlankNewEntry(level.ename[-1])
        n.flags |= IX_EXISTS
        (n.children_ofs,n.children_n) = level.write(f)
        level.parent.list.append(n)
        level = level.parent

    # create nodes down the tree
    while len(level.ename) < len(ename):
        level = Level(ename[:len(level.ename)+1], level)

    # are we in precisely the right place?
    assert(ename == level.ename)
    n = newentry or BlankNewEntry(ename and level.ename[-1] or None)
    (n.children_ofs,n.children_n) = level.write(f)
    if level.parent:
        level.parent.list.append(n)
    level = level.parent

    return level


class Entry:
    def __init__(self, basename, name):
        self.basename = str(basename)
        self.name = str(name)
        self.children_ofs = 0
        self.children_n = 0

    def __repr__(self):
        return ("(%s,0x%04x,%d,%d,%d,%d,%d,%s/%s,0x%04x,0x%08x/%d)" 
                % (self.name, self.dev,
                   self.ctime, self.mtime, self.uid, self.gid,
                   self.size, oct(self.mode), oct(self.gitmode),
                   self.flags, self.children_ofs, self.children_n))

    def packed(self):
        try:
            return struct.pack(INDEX_SIG,
                           self.dev, self.ctime, self.mtime, 
                           self.uid, self.gid, self.size, self.mode,
                           self.gitmode, self.sha, self.flags,
                           self.children_ofs, self.children_n)
        except DeprecationWarning, e:
            log('pack error: %s (%r)\n' % (e, self))
            raise

    def from_stat(self, st, tstart):
        old = (self.dev, self.ctime, self.mtime,
               self.uid, self.gid, self.size, self.flags & IX_EXISTS)
        new = (st.st_dev, int(st.st_ctime), int(st.st_mtime),
               st.st_uid, st.st_gid, st.st_size, IX_EXISTS)
        self.dev = st.st_dev
        self.ctime = int(st.st_ctime)
        self.mtime = int(st.st_mtime)
        self.uid = st.st_uid
        self.gid = st.st_gid
        self.size = st.st_size
        self.mode = st.st_mode
        self.flags |= IX_EXISTS
        if int(st.st_ctime) >= tstart or old != new \
              or self.sha == EMPTY_SHA or not self.gitmode:
            self.invalidate()
        self._fixup()
        
    def _fixup(self):
        if self.uid < 0:
            self.uid += 0x100000000
        if self.gid < 0:
            self.gid += 0x100000000
        assert(self.uid >= 0)
        assert(self.gid >= 0)
        if self.mtime < -0x80000000:  # can happen in NTFS on 64-bit linux
            self.mtime = 0
        if self.ctime < -0x80000000:
            self.ctime = 0
        if self.mtime > 0x7fffffff:
            self.mtime = 0x7fffffff
        if self.ctime > 0x7fffffff:
            self.ctime = 0x7fffffff

    def is_valid(self):
        f = IX_HASHVALID|IX_EXISTS
        return (self.flags & f) == f

    def invalidate(self):
        self.flags &= ~IX_HASHVALID

    def validate(self, gitmode, sha):
        assert(sha)
        assert(gitmode)
        self.gitmode = gitmode
        self.sha = sha
        self.flags |= IX_HASHVALID|IX_EXISTS

    def exists(self):
        return not self.is_deleted()

    def sha_missing(self):
        return (self.flags & IX_SHAMISSING) or not (self.flags & IX_HASHVALID)
{'fraction_non_alphanumeric': 0.11888579387186629, 'fraction_numerical': 0.019944289693593315, 'mean_word_length': 1.9362119725220805, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 10, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2683338', 'n_tokens_mistral': 3218, 'n_tokens_neox': 3002, 'n_words': 699}