<commit_before>"""
Capability to manage and run ATS system level tests including
regression tests and user demos.

With inspiration, and in some places just stolen, from <NAME>'s
PFloTran regression test suite.

Author: <NAME> (<EMAIL>)
"""

from __future__ import print_function
from __future__ import division

import sys

if sys.hexversion < 0x02070000:
    print(70*"*")
    print("ERROR: The regression test manager requires python >= 2.7.x. ")
    print("It appears that you are running python {0}.{1}.{2}".format(
        sys.version_info[0], sys.version_info[1], sys.version_info[2]))
    print(70*"*")
    sys.exit(1)

import datetime
import os
import glob
#import pprint
import shutil
import subprocess
import textwrap
import time
import traceback
import distutils.spawn
import numpy
import collections
import h5py


aliases = dict()


def get_git_hash(directory):
    """Helper function to return the git hash of a repo."""
    import subprocess
    return subprocess.check_output(['git', 'describe', '--always'], cwd=directory).strip()

def version(executable):
    """Helper function to pull a version number from an executable.
    
    For now this is hard-coded and assumes standard ATS locations,
    which is VERY fragile, and may even provide incorrect info.
    This needs to be fixed to take the output of ats --version
    (which doesn't yet work, see ats ticket #12
    """

    try:
        ats_label = get_git_hash(os.environ['ATS_SRC_DIR'])
    except KeyError:
        ats_label = 'unknown'

    try:
        amanzi_label = get_git_hash(os.environ['AMANZI_SRC_DIR'])
    except KeyError:
        amanzi_label = 'unknown'
    v = "ATS: {0}\nAmanzi: {1}".format(ats_label, amanzi_label)
    return v



class NoCatchException(Exception):
    def __init__(self,*args,**kwargs):
        Exception.__init__(self,*args,**kwargs)

if sys.version_info[0] == 2:
    from ConfigParser import SafeConfigParser as config_parser
else:
    from configparser import ConfigParser as config_parser


class TestStatus(object):
    """
    Simple class to hold status info.
    """
    def __init__(self):
        self.fail = 0
        self.warning = 0
        self.error = 0
        self.skipped = 0
        self.test_count = 0

    def __str__(self):
        message = "fail = {0}\n".format(self.fail)
        message += "warning = {0}\n".format(self.warning)
        message += "error = {0}\n".format(self.error)
        message += "skipped = {0}\n".format(self.skipped)
        message += "test_count = {0}\n".format(self.test_count)
        return message


class RegressionTest(object):
    """
    Class to collect data about a test problem, run the problem, and
    compare the results to a known result.
    """
    # things to compare
    _TIME = "time"
    _TIMESTEPS = "timesteps"
    _DEFAULT = "default" # compare a variable
    
    # ways to compare it
    _ABSOLUTE = "absolute"
    _RELATIVE = "relative"
    _PERCENT = "percent"
    _DISCRETE = "discrete"

    # ways to measure what to compare
    _NORM = numpy.inf

    _SUCCESS = 0
    _RESERVED = [_TIME, _TIMESTEPS]
    
    def __init__(self, executable='ats', mpiexec=None, version=None, suffix=None):
        if suffix is not None:
            self._suffix = "."+suffix
        else:
            self._suffix = ".regression"
        
        # misc test parameters
        #        self._pprint = pprint.PrettyPrinter(indent=2)
        self._txtwrap = textwrap.TextWrapper(width=78, subsequent_indent=4*" ")
        self._executable = executable
        self._mpiexec = mpiexec
        self._version = version
        self._input_arg = "--xml_file="
        self._input_suffix = "xml"
        self._np = None
        self._timeout = 162.0
        self._skip_check_gold = False
        self._check_performance = False
        self._num_failed = 0
        self._test_name = None

        # assign default tolerances for different classes of variables
        # absolute min and max thresholds for determining whether to
        # compare to baseline, i.e. if (min_threshold <= abs(value) <=
        # max_threshold) then compare values. By default we use the
        # python definitions for this platform
        self._default_tolerance = {}
        self._default_tolerance[self._TIME] = [1.e-12, self._ABSOLUTE, \
                                               0.0, sys.float_info.max]
        self._default_tolerance[self._DISCRETE] = [0, self._ABSOLUTE, 0, sys.maxsize]
        
        self._default_tolerance[self._DEFAULT] = [1.0e-10, self._RELATIVE, \
                                                  0.0, sys.float_info.max]
        self._eps = 1.e-14

        # dictionary indexed by domain, each of which is a dictionary of
        # (key, tolerance) pairs
        self._criteria = {}
        self._file_prefix = "checkpoint"
        self._file_suffix = ".h5"
        self._file_format = self._file_prefix + "{0:05d}" + self._file_suffix

    def __str__(self):
        message = "  {0} :\n".format(self.name())
        message += "    timeout = {0}\n".format(self._timeout)
        message += "    np = {0}\n".format(self._np)
        message += "    ATS args :\n"
        message += "        exec  : {0}\n".format(self._executable)
        message += "        input : {0}../{1}.{2}\n".format(
            self._input_arg,self._test_name,self._input_suffix)
        for domain,criteria in self._criteria.iteritems():
            message += "    test criteria on \"{0}\" :\n".format(domain)
            for key,tol in criteria.iteritems():
                message += "        {0} : {1} [{2}] : {3} <= abs(value) <= {4}\n".format(
                    key, tol[0], tol[1], tol[2], tol[3])
        return message

    def setup(self, cfg_criteria, test_data, timeout, check_performance, testlog):
        """
        Setup the test object

        cfg_criteria - dict from cfg file, all tests in file
        test_data - dict from cfg file, test specific
        timeout - list(?) from command line option
        check_performance - bool from command line option
        """
        self._test_name = test_data.pop('name')
        self._np = test_data.pop('np', None)
        self._skip_check_gold = test_data.pop('skip_check_gold', None)
        self._check_performance = check_performance

        # timeout : preference (1) command line (2) test data (3) class default
        self._timeout = float(cfg_criteria.pop('timeout', self._timeout))
        self._timeout = float(test_data.pop('timeout', self._timeout))
        if timeout is not None:
            self._timeout = float(timeout[0])

        # pop a default and/or default discrete
        default = test_data.pop(self._DEFAULT,
                                cfg_criteria.pop(self._DEFAULT, None))
        if default is not None:
            tol = self._validate_tolerance(self._DEFAULT, default)
            self._default_tolerance[self._DEFAULT] = tol

        discrete = test_data.pop(self._DISCRETE,
                                cfg_criteria.pop(self._DISCRETE, None))
        if discrete is not None:
            tol = self._validate_tolerance(self._DISCRETE, discrete)
            self._default_tolerance[self._DISCRETE] = tol
            
        # requested criteria, skipping time and timesteps
        for key in set(cfg_criteria.keys() + test_data.keys()):
            if key != self._TIME and key != self._TIMESTEPS:
                self._set_criteria(key, cfg_criteria, test_data)

        # always check that write simulation time are the same
        self._set_criteria(self._TIME, cfg_criteria, test_data)

    def name(self):
        return self._test_name

    def dirname(self, gold=False):
        suffix = self._suffix
        if gold:
            suffix = suffix + ".gold"
        return self.name() + suffix        

    def filenames(self, dirname):
        fname_list = sorted(glob.glob(os.path.join(dirname, self._file_prefix+"*"+self._file_suffix)))
        final_name = os.path.join(dirname, 'checkpoint_final.h5')
        if final_name in fname_list:
            fname_list.remove(final_name)
        return fname_list

    def run(self, dry_run, status, testlog):
        """
        Run the test.
        """
        self._cleanup_generated_files()
        self._run_test(self.name(), dry_run, status,
                       testlog)

    def _run_test(self, test_name, dry_run, status, testlog):
        """
        Build up the run command, including mpiexec, np, ats,
        input file, output file. Then run the job as a subprocess.

        * NOTE(bja) - starting in python 3.3, we can use:

          subprocess.Popen(...).wait(timeout)

          to catch hanging jobs, but for python < 3.3 we have to
          manually manage the timeout...?
        """
        command = []
        if self._np is not None:
            if self._mpiexec:
                command.append(self._mpiexec)
                command.append("-np")
                command.append(self._np)
            else:
                # parallel test, but don't have mpiexec, we mark the
                # test as skipped and bail....
                message = self._txtwrap.fill(
                    "WARNING : mpiexec was not provided for a parallel test '{0}'.\n"
                    "This test was skipped!".format(self.name()))
                print(message, file=testlog)
                status.skipped = 1
                return None

        command.append(self._executable)
        command.append("{0}../{1}.{2}".format(self._input_arg,test_name,self._input_suffix))

        test_directory = os.getcwd()
        run_directory = os.path.join(test_directory, self.dirname())
        os.mkdir(run_directory)
        os.chdir(run_directory)
        with open('ats_version.txt', 'w') as fid:
            fid.write(self._version)
        
        print("    cd {0}".format(run_directory), file=testlog)
        print("    {0}".format(" ".join(command)), file=testlog)

        if not dry_run:
            run_stdout = open(test_name + ".stdout", 'w')
            start = time.time()
            proc = subprocess.Popen(command,
                                    shell=False,
                                    stdout=run_stdout,
                                    stderr=run_stdout)
            while proc.poll() is None:
                time.sleep(0.1)
                if time.time() - start > self._timeout:
                    proc.kill()
                    time.sleep(0.1)
                    message = self._txtwrap.fill(
                        "ERROR: job '{0}' has exceeded timeout limit of "
                        "{1} seconds.".format(test_name, self._timeout))
                    print(''.join(['\n', message, '\n']), file=testlog)
            finish = time.time()
            print("    # {0} : run time : {1:.2f} seconds".format(test_name, finish - start), file=testlog)
            ierr_status = abs(proc.returncode)
            run_stdout.close()

            if ierr_status != self._SUCCESS:
                status.fail = 1
                message = self._txtwrap.fill(
                    "FAIL : {name} : {execute} return an error "
                    "code ({status}) indicating the simulation may have "
                    "failed. Please check '{name}.stdout' "
                    "for error messages (included below).".format(
                        execute=self._executable, name=test_name, status=ierr_status))

                print("".join(['\n', message, '\n']), file=testlog)
                print("~~~~~ {0}.stdout ~~~~~".format(test_name), file=testlog)
                try:
                    with open("{0}.stdout".format(test_name), 'r') as tempfile:
                        shutil.copyfileobj(tempfile, testlog)
                except Exception as e:
                    print("   Error opening file: {0}.stdout\n    {1}".format(test_name, e))
                print("~~~~~~~~~~", file=testlog)

        os.chdir(test_directory)

    def _cleanup_generated_files(self):
        """Cleanup old generated files that may be hanging around from a
        previous run.
        """
        run_dir = self.dirname(False)
        shutil.rmtree(run_dir, ignore_errors=True)


    def check(self, status, testlog):
        """
        Check the test results against the gold standard
        """
        self._check_gold(status, testlog)

    def update(self, status, testlog):
        """
        Update the gold standard test results to the current
        output. Both the current regression output and a gold file
        must exist.
        """
        gold_dir = self.dirname(True)
        run_dir = self.dirname(False)
        old_gold_dir = gold_dir+".old"

        # verify that the gold file exists
        if not os.path.isdir(gold_dir):
            print("ERROR: test '{0}' results cannot be updated "
                  "because a gold directory does not "
                  "exist!".format(self.name()), file=testlog)
            status.error = 1
                
        # verify that the regression directory exists
        if not os.path.isdir(run_dir):
            print("ERROR: test '{0}' results cannot be updated "
                  "because no regression run directory "
                  "exists!".format(self.name()), file=testlog)
            status.error = 1

        # check that the regression file was created in the regression directory
        if (not status.error and not status.fail):
            reg_filenames = self.filenames(run_dir)
            if len(reg_filenames) == 0:
                print("ERROR: run for "
                      "test '{0}' did not create the needed regression file(s), "
                      "not updating!".format(self.name(),
                                             reg_name), file=testlog)
                status.fail = 1

        if not status.error and not status.fail:
            # remove old-old gold
            if os.path.isdir(old_gold_dir):
                try:
                    shutil.rmtree(old_gold_dir)
                except Exception as error:
                    message = str(error)
                    message += "\nFAIL : Could not remove old gold '{0}'. ".format(old_gold_dir)
                    message += "Please remove the directory manually!"
                    message += "    rm -rf {0}".format(old_gold_dir)
                    print(message, file=testlog)
                    status.fail = 1

            # move gold -> old_gold
            if not status.fail:
                try:
                    os.rename(gold_dir, old_gold_dir)
                except Exception as error:
                    message = str(error)
                    message += "\nFAIL : Could not back up gold '{0}' as '{1}'. ".format(
                        gold_dir, old_gold_dir)
                    message += "Please rename or remove the gold directory manually!"
                    message += "    mv {0} {1}".format(gold_dir, old_gold_dir)
                    print(message, file=testlog)
                    status.fail = 1

            if not status.fail:
                # move run -> gold
                try:
                    os.rename(run_dir, gold_dir)
                except Exception as error:
                    message = str(error)
                    message += "\nFAIL : Could not move '{0}' to '{1}'. ".format(
                        run_dir, gold_dir)
                    message += "Please rename the directory manually!"
                    message += "    mv {0} {1}".format(run_dir, gold_dir)
                    print(message, file=testlog)
                    status.fail = 1

        print("done", file=testlog)

    def new_test(self, status, testlog):
        """
        A new test does not have a gold standard regression test. We
        will check to see if a gold standard file exists (an error),
        then create the gold file by copying the current regression
        file to gold.
        """
        gold_dir = self.dirname(True)
        run_dir = self.dirname(False)

        # verify that the gold file does not exisit
        if os.path.isdir(gold_dir) or os.path.isfile(gold_dir):
            print("ERROR: test '{0}' was classified as new, "
                               "but a gold file already "
                               "exists!".format(self.name()), file=testlog)
            status.error = 1

        # verify that the run directory exists
        if not os.path.isdir(run_dir):
            print("ERROR: test '{0}' results cannot be new "
                  "because no run directory "
                  "exists!".format(self.name()), file=testlog)
            status.error = 1
        
        # verify that the run directory created good files
        if (not status.error and not status.fail):
            reg_filenames = self.filenames(run_dir)
            if len(reg_filenames) == 0:
                print("ERROR: run for "
                      "test '{0}' did not create the needed regression file "
                      "'{1}', not making new gold!".format(self.name(),
                                                           reg_name), file=testlog)
                status.fail = 1

        # move the run to gold
        if not status.error and not status.fail:
            print("  creating gold directory '{0}'... ".format(self.name()),
                  file=testlog)
            try:
                os.rename(run_dir, gold_dir)
            except Exception as error:
                message = str(error)
                message += "\nFAIL : Could not move '{0}' to '{1}'. ".format(
                    run_dir, gold_dir)
                message += "Please rename the directory manually!"
                message += "    mv {0} {1}".format(run_dir, gold_dir)
                print(message, file=testlog)
                status.fail = 1

        print("done", file=testlog)

<commit_msg>fixes to test_manager to better support adding new tests, forcing the timestep structure, commiting results, etc<commit_after>"""
Capability to manage and run ATS system level tests including
regression tests and user demos.

With inspiration, and in some places just stolen, from <NAME>'s
PFloTran regression test suite.

Author: <NAME> (<EMAIL>)
"""

from __future__ import print_function
from __future__ import division

import sys

if sys.hexversion < 0x02070000:
    print(70*"*")
    print("ERROR: The regression test manager requires python >= 2.7.x. ")
    print("It appears that you are running python {0}.{1}.{2}".format(
        sys.version_info[0], sys.version_info[1], sys.version_info[2]))
    print(70*"*")
    sys.exit(1)

import datetime
import os
import glob
#import pprint
import shutil
import subprocess
import textwrap
import time
import traceback
import distutils.spawn
import numpy
import collections
import h5py

aliases = dict()


def get_git_hash(directory):
    """Helper function to return the git hash of a repo."""
    import subprocess
    return subprocess.check_output(['git', 'describe', '--always'], cwd=directory).strip()

def version(executable):
    """Helper function to pull a version number from an executable.
    
    For now this is hard-coded and assumes standard ATS locations,
    which is VERY fragile, and may even provide incorrect info.
    This needs to be fixed to take the output of ats --version
    (which doesn't yet work, see ats ticket #12
    """

    try:
        ats_label = get_git_hash(os.environ['ATS_SRC_DIR'])
    except KeyError:
        ats_label = 'unknown'

    try:
        amanzi_label = get_git_hash(os.environ['AMANZI_SRC_DIR'])
    except KeyError:
        amanzi_label = 'unknown'
    v = "ATS: {0}\nAmanzi: {1}".format(ats_label, amanzi_label)
    return v



class NoCatchException(Exception):
    def __init__(self,*args,**kwargs):
        Exception.__init__(self,*args,**kwargs)

if sys.version_info[0] == 2:
    from ConfigParser import SafeConfigParser as config_parser
else:
    from configparser import ConfigParser as config_parser


class TestStatus(object):
    """
    Simple class to hold status info.
    """
    def __init__(self):
        self.fail = 0
        self.warning = 0
        self.error = 0
        self.skipped = 0
        self.test_count = 0

    def __str__(self):
        message = "fail = {0}\n".format(self.fail)
        message += "warning = {0}\n".format(self.warning)
        message += "error = {0}\n".format(self.error)
        message += "skipped = {0}\n".format(self.skipped)
        message += "test_count = {0}\n".format(self.test_count)
        return message


class RegressionTest(object):
    """
    Class to collect data about a test problem, run the problem, and
    compare the results to a known result.
    """
    # things to compare
    _TIME = "time"
    _TIMESTEPS = "timesteps"
    _DEFAULT = "default" # compare a variable
    
    # ways to compare it
    _ABSOLUTE = "absolute"
    _RELATIVE = "relative"
    _PERCENT = "percent"
    _DISCRETE = "discrete"

    # ways to measure what to compare
    _NORM = numpy.inf

    _SUCCESS = 0
    _RESERVED = [_TIME, _TIMESTEPS]
    
    def __init__(self, executable='ats', mpiexec=None, version=None, suffix=None):
        if suffix is not None:
            self._suffix = "."+suffix
        else:
            self._suffix = ".regression"
        
        # misc test parameters
        #        self._pprint = pprint.PrettyPrinter(indent=2)
        self._txtwrap = textwrap.TextWrapper(width=78, subsequent_indent=4*" ")
        self._executable = executable
        self._mpiexec = mpiexec
        self._version = version
        self._input_arg = "--xml_file="
        self._input_suffix = "xml"
        self._np = None
        self._timeout = 162.0
        self._skip_check_gold = False
        self._check_performance = False
        self._num_failed = 0
        self._test_name = None

        # assign default tolerances for different classes of variables
        # absolute min and max thresholds for determining whether to
        # compare to baseline, i.e. if (min_threshold <= abs(value) <=
        # max_threshold) then compare values. By default we use the
        # python definitions for this platform
        self._default_tolerance = {}
        self._default_tolerance[self._TIME] = [1.e-12, self._ABSOLUTE, \
                                               0.0, sys.float_info.max]
        self._default_tolerance[self._DISCRETE] = [0, self._ABSOLUTE, 0, sys.maxsize]
        
        self._default_tolerance[self._DEFAULT] = [1.0e-10, self._RELATIVE, \
                                                  0.0, sys.float_info.max]
        self._eps = 1.e-14

        # dictionary indexed by domain, each of which is a dictionary of
        # (key, tolerance) pairs
        self._criteria = {}
        self._file_prefix = "checkpoint"
        self._file_suffix = ".h5"
        self._file_format = self._file_prefix + "{0:05d}" + self._file_suffix

    def __str__(self):
        message = "  {0} :\n".format(self.name())
        message += "    timeout = {0}\n".format(self._timeout)
        message += "    np = {0}\n".format(self._np)
        message += "    ATS args :\n"
        message += "        exec  : {0}\n".format(self._executable)
        message += "        input : {0}../{1}.{2}\n".format(
            self._input_arg,self._test_name,self._input_suffix)
        for domain,criteria in self._criteria.iteritems():
            message += "    test criteria on \"{0}\" :\n".format(domain)
            for key,tol in criteria.iteritems():
                message += "        {0} : {1} [{2}] : {3} <= abs(value) <= {4}\n".format(
                    key, tol[0], tol[1], tol[2], tol[3])
        return message

    def setup(self, cfg_criteria, test_data, timeout, check_performance, testlog):
        """
        Setup the test object

        cfg_criteria - dict from cfg file, all tests in file
        test_data - dict from cfg file, test specific
        timeout - list(?) from command line option
        check_performance - bool from command line option
        """
        self._test_name = test_data.pop('name')
        self._np = test_data.pop('np', None)
        self._skip_check_gold = test_data.pop('skip_check_gold', None)
        self._check_performance = check_performance

        # timeout : preference (1) command line (2) test data (3) class default
        self._timeout = float(cfg_criteria.pop('timeout', self._timeout))
        self._timeout = float(test_data.pop('timeout', self._timeout))
        if timeout is not None:
            self._timeout = float(timeout[0])

        # pop a default and/or default discrete
        default = test_data.pop(self._DEFAULT,
                                cfg_criteria.pop(self._DEFAULT, None))
        if default is not None:
            tol = self._validate_tolerance(self._DEFAULT, default)
            self._default_tolerance[self._DEFAULT] = tol

        discrete = test_data.pop(self._DISCRETE,
                                cfg_criteria.pop(self._DISCRETE, None))
        if discrete is not None:
            tol = self._validate_tolerance(self._DISCRETE, discrete)
            self._default_tolerance[self._DISCRETE] = tol
            
        # requested criteria, skipping time and timesteps
        for key in set(cfg_criteria.keys() + test_data.keys()):
            if key != self._TIME and key != self._TIMESTEPS:
                self._set_criteria(key, cfg_criteria, test_data)

        # always check that write simulation time are the same
        self._set_criteria(self._TIME, cfg_criteria, test_data)

    def name(self):
        return self._test_name

    def dirname(self, gold=False):
        suffix = self._suffix
        if gold:
            suffix = suffix + ".gold"
        return self.name() + suffix        

    def filenames(self, dirname):
        fname_list = sorted(glob.glob(os.path.join(dirname, self._file_prefix+"*"+self._file_suffix)))
        final_name = os.path.join(dirname, 'checkpoint_final.h5')
        if final_name in fname_list:
            fname_list.remove(final_name)
        return fname_list

    def run(self, dry_run, status, testlog):
        """
        Run the test.
        """
        self._cleanup_generated_files()
        self._run_test(self.name(), dry_run, status,
                       testlog)

    def _run_test(self, test_name, dry_run, status, testlog):
        """
        Build up the run command, including mpiexec, np, ats,
        input file, output file. Then run the job as a subprocess.

        * NOTE(bja) - starting in python 3.3, we can use:

          subprocess.Popen(...).wait(timeout)

          to catch hanging jobs, but for python < 3.3 we have to
          manually manage the timeout...?
        """
        command = []
        if self._np is not None:
            if self._mpiexec:
                command.append(self._mpiexec)
                command.append("-np")
                command.append(self._np)
            else:
                # parallel test, but don't have mpiexec, we mark the
                # test as skipped and bail....
                message = self._txtwrap.fill(
                    "WARNING : mpiexec was not provided for a parallel test '{0}'.\n"
                    "This test was skipped!".format(self.name()))
                print(message, file=testlog)
                status.skipped = 1
                return None

        command.append(self._executable)
        command.append("{0}../{1}.{2}".format(self._input_arg,test_name,self._input_suffix))

        test_directory = os.getcwd()
        run_directory = os.path.join(test_directory, self.dirname())
        os.mkdir(run_directory)
        os.chdir(run_directory)
        with open('ats_version.txt', 'w') as fid:
            fid.write(self._version)
        
        print("    cd {0}".format(run_directory), file=testlog)
        print("    {0}".format(" ".join(command)), file=testlog)

        if not dry_run:
            run_stdout = open(test_name + ".stdout", 'w')
            start = time.time()
            proc = subprocess.Popen(command,
                                    shell=False,
                                    stdout=run_stdout,
                                    stderr=run_stdout)
            while proc.poll() is None:
                time.sleep(0.1)
                if time.time() - start > self._timeout:
                    proc.kill()
                    time.sleep(0.1)
                    message = self._txtwrap.fill(
                        "ERROR: job '{0}' has exceeded timeout limit of "
                        "{1} seconds.".format(test_name, self._timeout))
                    print(''.join(['\n', message, '\n']), file=testlog)
            finish = time.time()
            print("    # {0} : run time : {1:.2f} seconds".format(test_name, finish - start), file=testlog)
            ierr_status = abs(proc.returncode)
            run_stdout.close()

            if ierr_status != self._SUCCESS:
                status.fail = 1
                message = self._txtwrap.fill(
                    "FAIL : {name} : {execute} return an error "
                    "code ({status}) indicating the simulation may have "
                    "failed. Please check '{name}.stdout' "
                    "for error messages (included below).".format(
                        execute=self._executable, name=test_name, status=ierr_status))

                print("".join(['\n', message, '\n']), file=testlog)
                print("~~~~~ {0}.stdout ~~~~~".format(test_name), file=testlog)
                try:
                    with open("{0}.stdout".format(test_name), 'r') as tempfile:
                        shutil.copyfileobj(tempfile, testlog)
                except Exception as e:
                    print("   Error opening file: {0}.stdout\n    {1}".format(test_name, e))
                print("~~~~~~~~~~", file=testlog)

        os.chdir(test_directory)

    def _cleanup_generated_files(self):
        """Cleanup old generated files that may be hanging around from a
        previous run.
        """
        run_dir = self.dirname(False)
        shutil.rmtree(run_dir, ignore_errors=True)


    def check(self, status, testlog):
        """
        Check the test results against the gold standard
        """
        self._check_gold(status, testlog)

    def update(self, status, testlog):
        """
        Update the gold standard test results to the current
        output. Both the current regression output and a gold file
        must exist.
        """
        gold_dir = self.dirname(True)
        run_dir = self.dirname(False)
        old_gold_dir = gold_dir+".old"

        # verify that the gold file exists
        if not os.path.isdir(gold_dir):
            print("ERROR: test '{0}' results cannot be updated "
                  "because a gold directory does not "
                  "exist!".format(self.name()), file=testlog)
            status.error = 1
                
        # verify that the regression directory exists
        if not os.path.isdir(run_dir):
            print("ERROR: test '{0}' results cannot be updated "
                  "because no regression run directory "
                  "exists!".format(self.name()), file=testlog)
            status.error = 1

        # check that the regression file was created in the regression directory
        if (not status.error and not status.fail):
            reg_filenames = self.filenames(run_dir)
            if len(reg_filenames) == 0:
                print("ERROR: run for "
                      "test '{0}' did not create the needed regression file(s), "
                      "not updating!".format(self.name(),
                                             reg_name), file=testlog)
                status.fail = 1

        if not status.error and not status.fail:
            # remove old-old gold
            if os.path.isdir(old_gold_dir):
                try:
                    shutil.rmtree(old_gold_dir)
                except Exception as error:
                    message = str(error)
                    message += "\nFAIL : Could not remove old gold '{0}'. ".format(old_gold_dir)
                    message += "Please remove the directory manually!"
                    message += "    rm -rf {0}".format(old_gold_dir)
                    print(message, file=testlog)
                    status.fail = 1

            # move gold -> old_gold
            if not status.fail:
                try:
                    os.rename(gold_dir, old_gold_dir)
                except Exception as error:
                    message = str(error)
                    message += "\nFAIL : Could not back up gold '{0}' as '{1}'. ".format(
                        gold_dir, old_gold_dir)
                    message += "Please rename or remove the gold directory manually!"
                    message += "    mv {0} {1}".format(gold_dir, old_gold_dir)
                    print(message, file=testlog)
                    status.fail = 1

            if not status.fail:
                # move run -> gold
                try:
                    os.rename(run_dir, gold_dir)
                except Exception as error:
                    message = str(error)
                    message += "\nFAIL : Could not move '{0}' to '{1}'. ".format(
                        run_dir, gold_dir)
                    message += "Please rename the directory manually!"
                    message += "    mv {0} {1}".format(run_dir, gold_dir)
                    print(message, file=testlog)
                    status.fail = 1

        print("done", file=testlog)

    def new_test(self, status, testlog):
        """
        A new test does not have a gold standard regression test. We
        will check to see if a gold standard file exists (an error),
        then create the gold file by copying the current regression
        file to gold.
        """
        # this needs some extra things...
        import parse_logfile
        import amanzi_xml.utils.io as aio
        import amanzi_xml.utils.search as asearch
        import amanzi_xml.utils.errors as aerrors
        
        gold_dir = self.dirname(True)
        run_dir = self.dirname(False)

        # verify that the gold file does not exisit
        if os.path.isdir(gold_dir) or os.path.isfile(gold_dir):
            print("ERROR: test '{0}' was classified as new, "
                               "but a gold file already "
                               "exists!".format(self.name()), file=testlog)
            status.error = 1

        # verify that the run directory exists
        if not os.path.isdir(run_dir):
            print("ERROR: test '{0}' results cannot be new "
                  "because no run directory "
                  "exists!".format(self.name()), file=testlog)
            status.error = 1
        
        # verify that the run directory created good files
        if (not status.error and not status.fail):
            reg_filenames = self.filenames(run_dir)
            if len(reg_filenames) == 0:
                print("ERROR: run for "
                      "test '{0}' did not create the needed regression file "
                      "'{1}', not making new gold!".format(self.name(),
                                                           reg_name), file=testlog)
                status.fail = 1

        if not status.error and not status.fail:
            # move the run to gold
            print("  creating gold directory '{0}'... ".format(self.name()),
                  file=testlog)
            try:
                os.rename(run_dir, gold_dir)
            except Exception as error:
                message = str(error)
                message += "\nFAIL : Could not move '{0}' to '{1}'. ".format(
                    run_dir, gold_dir)
                message += "Please rename the directory manually!"
                message += "    mv {0} {1}".format(run_dir, gold_dir)
                print(message, file=testlog)
                status.fail = 1

            # build the dt history
            with open(os.path.join(gold_dir, self.name()+'.stdout'),'r') as fid:
                good, bad = parse_logfile.parse_logfile(fid)

            if not os.path.isdir('data'):
                os.mkdir('data')

            with h5py.File(os.path.join('data', "{0}_dts.h5".format(self.name())), 'w') as fid:
                fid.create_dataset("timesteps", data=86400.*good[:,2]) # note conversion from days back to seconds
            print("Wrote file: data/{0}_dts.h5".format(self.name()), file=testlog)

            # build a new xml file with these specific timestep history
            print("Renaming: {0}.xml to {0}_orig.xml".format(self.name()), file=testlog)
            os.rename(self.name()+".xml", self.name()+"_orig.xml")

            xml = aio.fromFile(self.name()+"_orig.xml", True)

            # -- checkpoint by list of cycles
            filenames = self.filenames(gold_dir)
            chp_cycles = [int(os.path.split(f)[-1][10:-3]) for f in filenames]
            try:
                chp = asearch.getElementByNamePath(xml, "checkpoint")
            except aerrors.MissingXMLError:
                pass
            else:
                # empty the checkpoint list
                for i in range(len(chp)):
                    chp.pop(chp[0].get('name'))

                chp.setParameter('cycles', 'Array(int)', chp_cycles)

            # -- update timestep controller, nonlinear solvers
            for ti in asearch.generateElementByNamePath(xml, "time integrator"):
                asearch.generateElementByNamePath(ti, "limit iterations").next().setValue(100)
                asearch.generateElementByNamePath(ti, "diverged tolerance").next().setValue(1.e10)

                asearch.getElementByNamePath(ti, "timestep controller type").setValue("from file")
                ts_hist = ti.sublist("timestep controller from file parameters")
                ts_hist.setParameter("file name", "string", "../data/{0}_dts.h5".format(self.name()))

            # -- write the new xml
            print("Writing: {0}.xml".format(self.name()), file=testlog)
            aio.toFile(xml, '{0}.xml'.format(self.name()))

            # clean the gold directory
            filenames.append(os.path.join(gold_dir, 'ats_version.txt'))
            for f in os.listdir(gold_dir):
                if not os.path.join(gold_dir, f) in filenames:
                    os.remove(os.path.join(gold_dir, f))

            # git add stuff
            filenames.append('{}_orig.xml'.format(self.name()))
            filenames.append('{}.xml'.format(self.name()))
            filenames.append(os.path.join('data','{}_dts.h5'.format(self.name())))
            msg = subprocess.check_output(['git', 'add', '-f',]+filenames)
            print(msg, file=testlog)

        print("done", file=testlog)


{'fraction_non_alphanumeric': 0.10852713178294573, 'fraction_numerical': 0.00847304849468181, 'mean_word_length': 1.8119342457817365, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '494797', 'n_tokens_mistral': 11161, 'n_tokens_neox': 10613, 'n_words': 2965}