<commit_before>from pickle import dump
from popen2 import popen2
import os
import time

class ClusterTools(object):
    cluster_output_filename="cluster_output.txt"
    cluster_error_filename="cluster_error.txt"
    
    @staticmethod
    def submit_experiments(experiment_list, cluster_command="run_experiment.py"):
        
        filenames=[]
        for experiment in experiment_list:
            filename=experiment.foldername+"experiment_instance.bin"
            filenames.append(filename)
            f=open(filename, 'w')
            dump(experiment, f)
            f.close()
            
        for i in range(len(filenames)):
            commands=[]
            commands.append("export PATH=$PATH:/nfs/home1/ucabjga/opt/epd/bin")
            commands.append("export PYTHONPATH=/nfs/home1/ucabhst/mcmc-hammer")
            commands.append("nice -n 10 python " + cluster_command + " " + filenames[i])
            
            job_name = filenames[i].split(os.sep)[-2].split(".")[0]
            walltime = "walltime=2:00:00"
            processors = "nodes=1:ppn=1"
            memory = "pmem=1gb"
            workdir = experiment.foldername
            output=experiment_list[i].foldername + ClusterTools.cluster_output_filename
            error=experiment_list[i].foldername + ClusterTools.cluster_error_filename
            
            job_string = """
            #PBS -S /bin/bash
            #PBS -N %s
            #PBS -l %s
            #PBS -l %s
            #PBS -l %s
            #PBS -o %s
            #PBS -e %s
            cd %s
            %s""" % (job_name, walltime, processors, memory, output, error, workdir, ";".join(commands))
        
            # send job_string to qsub
            outpipe, inpipe = popen2('qsub')
            print job_string
            inpipe.write(job_string)
            inpipe.close()
            
            print outpipe.read()
            time.sleep(0.1)<commit_msg>exports in job string again<commit_after>from pickle import dump
from popen2 import popen2
import os
import time

class ClusterTools(object):
    cluster_output_filename="cluster_output.txt"
    cluster_error_filename="cluster_error.txt"
    
    @staticmethod
    def submit_experiments(experiment_list, cluster_command="run_experiment.py"):
        
        filenames=[]
        for experiment in experiment_list:
            filename=experiment.foldername+"experiment_instance.bin"
            filenames.append(filename)
            f=open(filename, 'w')
            dump(experiment, f)
            f.close()
            
        for i in range(len(filenames)):
            command="nice -n 10 python " + cluster_command + " " + filenames[i]
            
            job_name = filenames[i].split(os.sep)[-2].split(".")[0]
            walltime = "walltime=2:00:00"
            processors = "nodes=1:ppn=1"
            memory = "pmem=1gb"
            workdir = experiment.foldername
            output=experiment_list[i].foldername + ClusterTools.cluster_output_filename
            error=experiment_list[i].foldername + ClusterTools.cluster_error_filename
            
            job_string = """
            #PBS -S /bin/bash
            #PBS -N %s
            #PBS -l %s
            #PBS -l %s
            #PBS -l %s
            #PBS -o %s
            #PBS -e %s
            export PATH=$PATH:/nfs/home1/ucabjga/opt/epd/bin
            export PYTHONPATH=/nfs/home1/ucabhst/mcmc-hammer
            cd %s
            %s""" % (job_name, walltime, processors, memory, output, error, workdir, command)
        
            # send job_string to qsub
            outpipe, inpipe = popen2('qsub')
            print job_string
            inpipe.write(job_string)
            inpipe.close()
            
            print outpipe.read()
            time.sleep(0.1)
{'fraction_non_alphanumeric': 0.10474934036939314, 'fraction_numerical': 0.010026385224274407, 'mean_word_length': 1.7957227138643068, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1762807', 'n_tokens_mistral': 1160, 'n_tokens_neox': 1120, 'n_words': 218}