<commit_before>
from __future__ import print_function

import subprocess as sp
import sys
import shlex
import shutil
import re
import os
import glob
import time
import threading

class ModelTestHelper(object):

    def __init__(self):

        self.my_path = os.path.dirname(os.path.realpath(__file__))
        self.lab_path = os.path.join(self.my_path, 'lab')
        self.bin_path = os.path.join(self.lab_path, 'bin')
        self.my_lock = threading.Lock()

    def make_paths(self, exp_name, run_num=0):
        paths = {}
        run_num = str(run_num).zfill(3)

        paths['exp'] = os.path.join(self.my_path, 'payu-experiments/access',
                                    exp_name)
        paths['archive'] = os.path.join(self.lab_path, 'archive', exp_name)
        paths['archive_link'] = os.path.join(paths['exp'], 'archive')
        paths['output'] = os.path.join(paths['archive'], 'output' + run_num)
        paths['restart'] = os.path.join(paths['archive'], 'restart' + run_num)
        paths['stdout'] = os.path.join(paths['output'], 'access.out')
        paths['stderr'] = os.path.join(paths['output'], 'access.err')

        paths['stdout_runtime'] = os.path.join(paths['exp'], 'access.out')
        paths['stderr_runtime'] = os.path.join(paths['exp'], 'access.err')

        return paths

    def post_build_checks(self, exes):

        for e in exes:
            assert(os.path.exists(e))

    def do_basic_build(self, exp):

        exp_path = os.path.join('payu-experiments/access/', exp)
        ret = self.build(exp_path)
        os.chdir(self.my_path)
        assert(ret == 0)

    def print_output(self, files):

        for file in files:
            if os.path.exists(file):
                with open(file, 'r') as f:
                    print(f.read())

    def get_most_recent_run_num(self, archive_path):
        """
        Look in the archive directory to find which build this is. 
        """

        dirs = glob.glob(archive_path + '/output*')
        dirs.sort()
<commit_msg>Update experiment dir.<commit_after>
from __future__ import print_function

import subprocess as sp
import sys
import shlex
import shutil
import re
import os
import glob
import time
import threading

class ModelTestHelper(object):

    def __init__(self):

        self.my_path = os.path.dirname(os.path.realpath(__file__))
        self.lab_path = os.path.join(self.my_path, 'lab')
        self.bin_path = os.path.join(self.lab_path, 'bin')
        self.my_lock = threading.Lock()

    def make_paths(self, exp_name, run_num=0):
        paths = {}
        run_num = str(run_num).zfill(3)

        paths['exp'] = os.path.join(self.my_path, 'experiments/access',
                                    exp_name)
        paths['archive'] = os.path.join(self.lab_path, 'archive', exp_name)
        paths['archive_link'] = os.path.join(paths['exp'], 'archive')
        paths['output'] = os.path.join(paths['archive'], 'output' + run_num)
        paths['restart'] = os.path.join(paths['archive'], 'restart' + run_num)
        paths['stdout'] = os.path.join(paths['output'], 'access.out')
        paths['stderr'] = os.path.join(paths['output'], 'access.err')

        paths['stdout_runtime'] = os.path.join(paths['exp'], 'access.out')
        paths['stderr_runtime'] = os.path.join(paths['exp'], 'access.err')

        return paths

    def post_build_checks(self, exes):

        for e in exes:
            assert(os.path.exists(e))

    def do_basic_build(self, exp):

        exp_path = os.path.join('experiments/access/', exp)
        ret = self.build(exp_path)
        os.chdir(self.my_path)
        assert(ret == 0)

    def print_output(self, files):

        for file in files:
            if os.path.exists(file):
                with open(file, 'r') as f:
                    print(f.read())

    def get_most_recent_run_num(self, archive_path):
        """
        Look in the archive directory to find which build this is. 
        """

        dirs = glob.glob(archive_path + '/output*')
        dirs.sort()

{'fraction_non_alphanumeric': 0.14342530449913, 'fraction_numerical': 0.0014914243102162564, 'mean_word_length': 2.5360281195079084, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6687221', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1276, 'n_words': 264}