<commit_before># coding: utf-8
"""
Payu: A generic driver for numerical models on the NCI computing cluster (vayu)
-------------------------------------------------------------------------------
Contact:    <NAME> (<EMAIL>)
"""

import os
import sys
import shutil as sh
import subprocess as sp
import grp
import getpass
import errno

# Environment module support on vayu
execfile('/opt/Modules/default/init/python')

# Counter environment variable names
counter_env = 'count'
max_counter_env = 'max'
archive_server = 'dcc.nci.org.au'

#==============================================================================
class Experiment(object):
    """A generic numerical experiment on vayu"""
    
    #---
    def __init__(self, **kwargs):
        self.model_name = None
        self.modules = None
        self.config_files = None
        
        # Script names
        self.model_script = kwargs.pop('model_script', 'model.py')
        self.collation_script = kwargs.pop('collate_script', 'collate.py')
    
    
    #---
    def set_counters(self):
        self.counter = int(os.environ.get(counter_env, 1))
        self.max_counter = int(os.environ.get(max_counter_env, self.counter))
        
        assert 0 < self.max_counter
        assert 0 < self.counter <= self.max_counter
    
    
    #---
    def load_modules(self):
        module('purge')
        for mod in self.modules:
            module('load', mod)
    
    
    #---
    def path_names(self, **kwargs):
        # A model name must be assigned
        assert self.model_name
        
        # Experiment name (used for directories)
        default_name = os.path.basename(os.getcwd())
        self.name = kwargs.pop('name', default_name)
        
        # Configuration path (input, config)
        default_config_path = os.getcwd()
        self.config_path = kwargs.pop('config', default_config_path)
        
        # Laboratory path (output)
        default_user = getpass.getuser()
        self.user_name = kwargs.pop('user', default_user)
        
        # Project group
        default_project = os.environ.get('PROJECT')
        self.project_name = kwargs.pop('project', default_project)
        
        # Output path
        default_lab_path = os.path.join('/','short', self.project_name,
                                        self.user_name, self.model_name)
        self.lab_path = kwargs.pop('output', default_lab_path)
        
        # Experiment subdirectories
        self.archive_path = os.path.join(self.lab_path, 'archive', self.name)
        self.bin_path = os.path.join(self.lab_path, 'bin')
        self.work_path = os.path.join(self.lab_path, 'work', self.name)
        
        # Symbolic path to work
        self.work_sym_path = os.path.join(self.config_path, 'work')
        self.archive_sym_path = os.path.join(self.config_path, 'archive')
        
        # Set group identifier for output
        self.archive_group = kwargs.pop('archive_group', None)
        
        # Executable path
        exec_name = kwargs.pop('exe', self.default_exec)
        self.exec_path = os.path.join(self.bin_path, exec_name)
        
        # External forcing path
        forcing_dir = kwargs.pop('forcing', None)
        if forcing_dir:
            # Test for absolute path
            if os.path.exists(forcing_dir):
                self.forcing_path = forcing_dir
            else:
                # Test for path relative to /lab_path/forcing
                rel_path = os.path.join(self.lab_path, 'forcing', forcing_dir)
                if os.path.exists(rel_path):
                    self.forcing_path = rel_path
                else:
                    # Forcing does not exist; raise some exception
                    sys.exit('Forcing data not found; aborting.')
        
        # Archival path
        self.run_dir = 'run%02i' % (self.counter,)
        self.run_path = os.path.join(self.archive_path, self.run_dir)
   

    #---
    def setup(self):
        mkdir_p(self.work_path)
        
        # Create symbolic link to work
        if not os.path.exists(self.work_sym_path):
            os.symlink(self.work_path, self.work_sym_path)
        
        for f in self.config_files:
            f_path = os.path.join(self.config_path, f)
            sh.copy(f_path, self.work_path)
    
    
    #---
    def archive(self, collate=True, mdss=False):
        mkdir_p(self.archive_path)
       
        # Create archive symlink
        if not os.path.exists(self.archive_sym_path):
            os.symlink(self.archive_path, self.archive_sym_path)
        
        # Remove work symlink
        if os.path.islink(self.work_sym_path):
            os.remove(self.work_sym_path)
        
        # Check if archive path already exists
        # TODO: Check before running, rather than archiving
        if os.path.exists(self.run_path):
            sys.exit('Archived path already exists; aborting.')
        sh.move(self.work_path, self.run_path)
        
        if self.archive_group:
            self.regroup()
        
        if collate:
            job_name = os.environ.get('PBS_JOBNAME', self.name)
            cmd = ['qsub', self.collation_script, '-v', '%s=%i'
                    % (counter_env, self.counter)]
            rc = sp.Popen(cmd).wait()
    
    
    #---
    def remote_archive(self):
        archive_address = '%s@%s' % (getpass.getuser(), archive_server)
       
        ssh_key_path = os.path.join(os.getenv('HOME'), '.ssh',
                                    'id_rsa_file_transfer')
        
        # Top-level path is determined by the SSH key
        # (Usually /projects/[group])
        
        # Remote mkdir is currently not possible, so any new subdirectories
        # must be created before auto-archival
        remote_path = os.path.join(self.model_name, self.user_name,
                                   self.name)
        
        # TODO: how to remove shell=True ?
        cmd = 'rsync -a --safe-links -e "ssh -i %s" %s %s:%s' % \
                (ssh_key_path, self.run_path, archive_address, remote_path)
        rc = sp.Popen(cmd, shell=True).wait()
        assert rc == 0
    
    
    #---
    def regroup(self):
        uid = os.getuid()
        gid = grp.getgrnam(self.archive_group).gr_gid
        
        os.lchown(self.archive_path, uid, gid)
        for root, dirs, files in os.walk(self.archive_path):
            for d in dirs:
                os.lchown(os.path.join(root, d), uid, gid)
            for f in files:
                os.lchown(os.path.join(root, f), uid, gid)
    
    
    #---
    def resubmit(self):
        assert self.counter < self.max_counter
        self.counter += 1
        cmd = ['qsub', self.model_script, '-v', '%s=%i,%s=%i'
                % (counter_env, self.counter, max_counter_env,
                   self.max_counter) ]
        # TODO: Remove .wait()?
        sp.Popen(cmd).wait()


#==============================================================================
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError, exc:
        if exc.errno != errno.EEXIST:
            raise
<commit_msg>Archive server switched to dc<commit_after># coding: utf-8
"""
Payu: A generic driver for numerical models on the NCI computing cluster (vayu)
-------------------------------------------------------------------------------
Contact:    <NAME> (<EMAIL>)
"""

import os
import sys
import shutil as sh
import subprocess as sp
import grp
import getpass
import errno

# Environment module support on vayu
execfile('/opt/Modules/default/init/python')

# Counter environment variable names
counter_env = 'count'
max_counter_env = 'max'
archive_server = 'dc.nci.org.au'

#==============================================================================
class Experiment(object):
    """A generic numerical experiment on vayu"""
    
    #---
    def __init__(self, **kwargs):
        self.model_name = None
        self.modules = None
        self.config_files = None
        
        # Script names
        self.model_script = kwargs.pop('model_script', 'model.py')
        self.collation_script = kwargs.pop('collate_script', 'collate.py')
    
    
    #---
    def set_counters(self):
        self.counter = int(os.environ.get(counter_env, 1))
        self.max_counter = int(os.environ.get(max_counter_env, self.counter))
        
        assert 0 < self.max_counter
        assert 0 < self.counter <= self.max_counter
    
    
    #---
    def load_modules(self):
        module('purge')
        for mod in self.modules:
            module('load', mod)
    
    
    #---
    def path_names(self, **kwargs):
        # A model name must be assigned
        assert self.model_name
        
        # Experiment name (used for directories)
        default_name = os.path.basename(os.getcwd())
        self.name = kwargs.pop('name', default_name)
        
        # Configuration path (input, config)
        default_config_path = os.getcwd()
        self.config_path = kwargs.pop('config', default_config_path)
        
        # Laboratory path (output)
        default_user = getpass.getuser()
        self.user_name = kwargs.pop('user', default_user)
        
        # Project group
        default_project = os.environ.get('PROJECT')
        self.project_name = kwargs.pop('project', default_project)
        
        # Output path
        default_lab_path = os.path.join('/','short', self.project_name,
                                        self.user_name, self.model_name)
        self.lab_path = kwargs.pop('output', default_lab_path)
        
        # Experiment subdirectories
        self.archive_path = os.path.join(self.lab_path, 'archive', self.name)
        self.bin_path = os.path.join(self.lab_path, 'bin')
        self.work_path = os.path.join(self.lab_path, 'work', self.name)
        
        # Symbolic path to work
        self.work_sym_path = os.path.join(self.config_path, 'work')
        self.archive_sym_path = os.path.join(self.config_path, 'archive')
        
        # Set group identifier for output
        self.archive_group = kwargs.pop('archive_group', None)
        
        # Executable path
        exec_name = kwargs.pop('exe', self.default_exec)
        self.exec_path = os.path.join(self.bin_path, exec_name)
        
        # External forcing path
        forcing_dir = kwargs.pop('forcing', None)
        if forcing_dir:
            # Test for absolute path
            if os.path.exists(forcing_dir):
                self.forcing_path = forcing_dir
            else:
                # Test for path relative to /lab_path/forcing
                rel_path = os.path.join(self.lab_path, 'forcing', forcing_dir)
                if os.path.exists(rel_path):
                    self.forcing_path = rel_path
                else:
                    # Forcing does not exist; raise some exception
                    sys.exit('Forcing data not found; aborting.')
        
        # Archival path
        self.run_dir = 'run%02i' % (self.counter,)
        self.run_path = os.path.join(self.archive_path, self.run_dir)
   

    #---
    def setup(self):
        mkdir_p(self.work_path)
        
        # Create symbolic link to work
        if not os.path.exists(self.work_sym_path):
            os.symlink(self.work_path, self.work_sym_path)
        
        for f in self.config_files:
            f_path = os.path.join(self.config_path, f)
            sh.copy(f_path, self.work_path)
    
    
    #---
    def archive(self, collate=True, mdss=False):
        mkdir_p(self.archive_path)
       
        # Create archive symlink
        if not os.path.exists(self.archive_sym_path):
            os.symlink(self.archive_path, self.archive_sym_path)
        
        # Remove work symlink
        if os.path.islink(self.work_sym_path):
            os.remove(self.work_sym_path)
        
        # Check if archive path already exists
        # TODO: Check before running, rather than archiving
        if os.path.exists(self.run_path):
            sys.exit('Archived path already exists; aborting.')
        sh.move(self.work_path, self.run_path)
        
        if self.archive_group:
            self.regroup()
        
        if collate:
            job_name = os.environ.get('PBS_JOBNAME', self.name)
            cmd = ['qsub', self.collation_script, '-v', '%s=%i'
                    % (counter_env, self.counter)]
            rc = sp.Popen(cmd).wait()
    
    
    #---
    def remote_archive(self):
        archive_address = '%s@%s' % (getpass.getuser(), archive_server)
       
        ssh_key_path = os.path.join(os.getenv('HOME'), '.ssh',
                                    'id_rsa_file_transfer')
        
        # Top-level path is determined by the SSH key
        # (Usually /projects/[group])
        
        # Remote mkdir is currently not possible, so any new subdirectories
        # must be created before auto-archival
        remote_path = os.path.join(self.model_name, self.user_name,
                                   self.name)
        
        # TODO: how to remove shell=True ?
        cmd = 'rsync -a --safe-links -e "ssh -i %s" %s %s:%s' % \
                (ssh_key_path, self.run_path, archive_address, remote_path)
        rc = sp.Popen(cmd, shell=True).wait()
        assert rc == 0
    
    
    #---
    def regroup(self):
        uid = os.getuid()
        gid = grp.getgrnam(self.archive_group).gr_gid
        
        os.lchown(self.archive_path, uid, gid)
        for root, dirs, files in os.walk(self.archive_path):
            for d in dirs:
                os.lchown(os.path.join(root, d), uid, gid)
            for f in files:
                os.lchown(os.path.join(root, f), uid, gid)
    
    
    #---
    def resubmit(self):
        assert self.counter < self.max_counter
        self.counter += 1
        cmd = ['qsub', self.model_script, '-v', '%s=%i,%s=%i'
                % (counter_env, self.counter, max_counter_env,
                   self.max_counter) ]
        # TODO: Remove .wait()?
        sp.Popen(cmd).wait()


#==============================================================================
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError, exc:
        if exc.errno != errno.EEXIST:
            raise

{'fraction_non_alphanumeric': 0.13995643926087262, 'fraction_numerical': 0.0011241481065130331, 'mean_word_length': 1.8576591045974704, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2661369', 'n_tokens_mistral': 4317, 'n_tokens_neox': 4221, 'n_words': 912}