<commit_before>import os
import os.path
import shutil

from base_funcs import *
from base import InhibitorObject

class InhibitorSnapshot(InhibitorObject):
    """
    Manage a single snapshot such that catalyst will be happy with it.

    TODO:
        - support git branches
    """

    def __init__(self, name, rev=None, **keywords):
        self.name = name

        settings_conf = {
            'snapshot': {
                'required_keys':    ['repo_type', 'src', 'type'],
                'valid_keys':       ['rev'],
                'config_init':      {'name':name}
            }
        }

        super(InhibitorSnapshot, self).__init__(
            settings_conf=settings_conf,
            **keywords)
        
        self.repodir        = path_join(self.base['repo_cache'], name ) + '/'
        self.snapdir        = self.base['snapshots']
        self.src            = self.snapshot['src']
        self.type           = self.snapshot['type']
        self.repo_type      = self.snapshot['repo_type']
        self.force          = 'force' in self.base and self.base['force'] or False
        self.rev            = rev
        self.snapfile       = None

        if not self.rev and 'rev' in self.snapshot['rev']:
            self.rev = self.snapshot['rev']

        if self.rev:
            self.snapfile = path_join(self.snapdir, '%s-%s.tar.bz2'
                % (self.name, self.rev))


        if not self.repo_type in ['svn', 'git']:
            raise InhibitorError('Unknown snapshot src repo_type:  \'%s\'' % self.src)

    def is_overlay(self):
        return self.snapshot['type'] == 'overlay'

    def cachedir(self):
        return path_join(
            self.base['snapshot_cache'],
            '%s-%s' % (self.name, self.rev),
            'tree')
        

    def current_cache(self):
        if not self.rev or not self.snapfile:
            return False
       
        if os.path.exists(self.snapfile) \
            and os.path.isdir(self.cachedir()) \
            and os.listdir(self.cachedir()):
            return True


    def run(self):
        if self.repo_type == 'git':
            self._git_create_snapshot()
        elif self.repo_type == 'svn':
            self._svn_create_snapshot()

        self.create_cachedir()


    def create_cachedir(self):
        base_dir = path_join(self.base['snapshot_cache'], '%s-%s' % (self.name, self.rev) )
        if os.path.exists(base_dir):
            shutil.rmtree(base_dir)
        
        os.makedirs(base_dir)
        os.symlink('tree', path_join(base_dir, 'portage'))
        os.symlink('tree', path_join(base_dir, 'overlay'))

        cmd('tar -xjpf %s -C %s/' % (self.snapfile, base_dir))
        if 'catalyst_support' in self.base and self.base['catalyst_support']:
            write_hashfile(base_dir, self.snapfile, {'md5':None}, dest_filename='catalyst-hash')


    def _git_create_snapshot(self):
        myenv={'GIT_DIR':path_join(self.repodir, '.git')}

        if not os.path.isdir( myenv['GIT_DIR'] ):
            if os.path.exists(self.repodir):
                warn('%s is not a git repository.  Removing.' % self.repodir)
                try:
                    shutil.rmtree(self.repodir)
                except OSError, e:
                    InhibitorError('Cannot clean directory: %s' % e)

            cmd('git clone %s %s' % (self.src, self.repodir))
        else:
            cmd('git pull', env=myenv)

        if self.rev == None:
            self.rev = file_getline( path_join(myenv['GIT_DIR'], 'refs', 'heads', 'master') )
            self.rev = self.rev[:7]
            self.snapfile = path_join(self.snapdir, '%s-%s.tar.bz2'
                % (self.name, self.rev))

        if not self.force and self.current_cache():
            info('Skipping archive step, %s already exists' % os.path.basename(self.snapfile))
            return

        if os.path.exists(self.snapfile):
            try:
                os.unlink(self.snapfile)
            except OSError, e:
                raise InhibitorError("Failed to remove %s: %s" % (self.snapfile, e))
                
        cmd('git archive --format=tar --prefix=tree/ %s | bzip2 --fast -f > %s'
            % (self.rev, self.snapfile), env=myenv)
      
        md5_hash = get_checksum(self.snapfile)
        write_hashfile(self.snapdir, self.snapfile, {'md5':md5_hash})


    def _svn_create_snapshot(self):
        svndir = path_join(self.repodir, '.svn')
        if not os.path.isdir(svndir):
            if os.path.exists(self.repodir):
                warn('%s is not a svn repository.  Removing.' % self.repodir)
                try:
                    shutil.rmtree(self.repodir)
                except OSError, e:
                    InhibitorError('Cannot clean directory: %s' %e)
            cmd('svn checkout %s %s' % (self.src, self.repodir))    

        if self.rev == None:
            cmd('svn up %s' % (self.repodir))
            self.rev = cmd_out("svn info %s | awk '/Revision/{print $2}'" % self.repodir)
            self.rev = 'r' + self.rev
            self.snapfile = path_join(self.snapdir, '%s-%s.tar.bz2'
                % (self.name, self.rev))
        else:
            if self.rev[0] != 'r':
                self.rev = 'r' + self.rev
            cmd('svn up -%s %s' % (self.rev, self.repodir))

        if not self.force and self.current_cache():
            info('Skipping archive step, %s already exists' % os.path.basename(self.snapfile))
            return
 
        cmd("cd %s;tar -cjf %s --exclude='.svn' --transform='s,^,tree/,' ./"
            % (self.repodir, self.snapfile) )


if __name__ == '__main__':
    import getopt
    name        = None
    src         = None
    repo_type   = None
    rev         = None
    force       = False
    configfile  = None

    usage =  """snapshot.py [ARGUMENTS]
Creates a snapshot from a specified subversion or git repository.

REQUIRED ARGUMENTS:
    -n, --name <name>       Snapshot name<commit_msg>Remove svn snapshot support.<commit_after>import os
import os.path
import shutil

from base_funcs import *
from base import InhibitorObject

class InhibitorSnapshot(InhibitorObject):
    """
    Manage a single snapshot such that catalyst will be happy with it.

    TODO:
        - support git branches
        - repos aside from git (rsync, then svn probably)
    """

    def __init__(self, name, rev=None, **keywords):
        self.name = name

        settings_conf = {
            'snapshot': {
                'required_keys':    ['repo_type', 'src', 'type'],
                'valid_keys':       ['rev'],
                'config_init':      {'name':name}
            }
        }

        super(InhibitorSnapshot, self).__init__(
            settings_conf=settings_conf,
            **keywords)
        
        self.repodir        = path_join(self.base['repo_cache'], name ) + '/'
        self.snapdir        = self.base['snapshots']
        self.src            = self.snapshot['src']
        self.type           = self.snapshot['type']
        self.repo_type      = self.snapshot['repo_type']
        self.force          = 'force' in self.base and self.base['force'] or False
        self.rev            = rev
        self.snapfile       = None

        if not self.rev and 'rev' in self.snapshot['rev']:
            self.rev = self.snapshot['rev']

        if self.rev:
            self.snapfile = path_join(self.snapdir, '%s-%s.tar.bz2'
                % (self.name, self.rev))


        if not self.repo_type in ['git']:
            raise InhibitorError('Unknown snapshot src repo_type:  \'%s\'' % self.src)

    def is_overlay(self):
        return self.snapshot['type'] == 'overlay'

    def cachedir(self):
        return path_join(
            self.base['snapshot_cache'],
            '%s-%s' % (self.name, self.rev),
            'tree')
        

    def current_cache(self):
        if not self.rev or not self.snapfile:
            return False
       
        if os.path.exists(self.snapfile) \
            and os.path.isdir(self.cachedir()) \
            and os.listdir(self.cachedir()):
            return True


    def run(self):
        if self.repo_type == 'git':
            self._git_create_snapshot()

        self.create_cachedir()


    def create_cachedir(self):
        base_dir = path_join(self.base['snapshot_cache'], '%s-%s' % (self.name, self.rev) )
        if os.path.exists(base_dir):
            shutil.rmtree(base_dir)
        
        os.makedirs(base_dir)
        os.symlink('tree', path_join(base_dir, 'portage'))
        os.symlink('tree', path_join(base_dir, 'overlay'))

        cmd('tar -xjpf %s -C %s/' % (self.snapfile, base_dir))
        if 'catalyst_support' in self.base and self.base['catalyst_support']:
            write_hashfile(base_dir, self.snapfile, {'md5':None}, dest_filename='catalyst-hash')


    def _git_create_snapshot(self):
        myenv={'GIT_DIR':path_join(self.repodir, '.git')}

        if not os.path.isdir( myenv['GIT_DIR'] ):
            if os.path.exists(self.repodir):
                warn('%s is not a git repository.  Removing.' % self.repodir)
                try:
                    shutil.rmtree(self.repodir)
                except OSError, e:
                    InhibitorError('Cannot clean directory: %s' % e)

            cmd('git clone %s %s' % (self.src, self.repodir))
        else:
            cmd('git pull', env=myenv)

        if self.rev == None:
            self.rev = file_getline( path_join(myenv['GIT_DIR'], 'refs', 'heads', 'master') )
            self.rev = self.rev[:7]
            self.snapfile = path_join(self.snapdir, '%s-%s.tar.bz2'
                % (self.name, self.rev))

        if not self.force and self.current_cache():
            info('Skipping archive step, %s already exists' % os.path.basename(self.snapfile))
            return

        if os.path.exists(self.snapfile):
            try:
                os.unlink(self.snapfile)
            except OSError, e:
                raise InhibitorError("Failed to remove %s: %s" % (self.snapfile, e))
                
        cmd('git archive --format=tar --prefix=tree/ %s | bzip2 --fast -f > %s'
            % (self.rev, self.snapfile), env=myenv)
      
        md5_hash = get_checksum(self.snapfile)
        write_hashfile(self.snapdir, self.snapfile, {'md5':md5_hash})


if __name__ == '__main__':
    import getopt
    name        = None
    src         = None
    repo_type   = None
    rev         = None
    force       = False
    configfile  = None

    usage =  """snapshot.py [ARGUMENTS]
Creates a snapshot from a specified subversion or git repository.

REQUIRED ARGUMENTS:
    -n, --name <name>       Snapshot name
{'fraction_non_alphanumeric': 0.1301775147928994, 'fraction_numerical': 0.0017845402460787076, 'mean_word_length': 1.813953488372093, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '854912', 'n_tokens_mistral': 3403, 'n_tokens_neox': 3308, 'n_words': 730}