<commit_before><?php
/**

 * Created by JetBrains PhpStorm.
 * User: alain_desilets
 * Date: 2013-10-02
 * Time: 2:12 PM
 * To change this template use File | Settings | File Templates.
 *
 * This class is used to run phpunit tests and compare the list of failures
 * and errors to those of a benchmark "normal" run.
 *
 * Use this class in situations where it's not practical for everyone to
 * keep all the tests "in the green" at all time, and to only commit code
 * that doesn't break any tests.
 *
 * With this class, you can tell if you have broken tests that were working
 * previously, or if you have fixed tests that were broken before.
 */

require_once('lib/debug/Tracer.php');

class TestRunnerWithBaseline {

    private $last_test_started = null;

    private $logname_stem = 'phpunit-log';
    public $action = 'run'; // run|update_baseline
    public $phpunit_options = '';
    public $help = false;
    public $diffs;

    function run()
    {
        global $tracer;

        $this->config_from_cmdline_options();

        if ($this->help) {
            $this->usage();
        }

        $this->run_tests();

        $this->print_diffs_with_baseline();

        if ($this->action == 'update_baseline')
        {
            $this->save_current_log_as_baseline();
        }

    }

    function run_tests()
    {
        global $tracer;

        $cmd_line = "../../vendor/bin/phpunit";
        if ($this->phpunit_options != '')
        {
            $cmd_line = "$cmd_line ".$this->phpunit_options;
        }

        $cmd_line = $cmd_line." --log-json \"".$this->logname_current()."\" .";

        print "\n=======\n === Executing phpunit as: '$cmd_line'\n=======\n\n";

        system($cmd_line);
    }

    function print_diffs_with_baseline()
    {
        global $tracer;

        echo "\n\nChecking for differences with baseline test logs...\n\n";

        $baseline_issues;
        if (file_exists($this->logname_baseline()))
        {
            $baseline_issues = $this->read_log_file($this->logname_baseline());
        }
        else
        {
            echo "=== WARNING: No baseline file exists. Assuming empty baseline.\n\n";
            $baseline_issues = $this->make_empty_issues_list();
        }

        $current_issues = $this->read_log_file($this->logname_current());

        $this->diffs = $this->compare_two_test_runs($baseline_issues, $current_issues);

        $nb_failures_introduced = count($this->diffs['failures_introduced']);
        $nb_failures_fixed = count($this->diffs['failures_fixed']);
        $nb_errors_introduced = count($this->diffs['errors_introduced']);
        $nb_errors_fixed = count($this->diffs['errors_fixed']);

        $total_diffs =
            $nb_failures_introduced + $nb_errors_introduced +
                $nb_failures_fixed + $nb_errors_fixed;

        if ($total_diffs > 0)
        {
            echo "\n\nThere were $total_diffs differences with baseline.\n

Below is a list of tests that differ from the baseline.
See above details about each error or failure.
";
            if ($nb_failures_introduced > 0)
            {
                echo "\nNb of new FAILURES: $nb_failures_introduced:\n";
                foreach ($this->diffs['failures_introduced'] as $an_issue)
                {
                    echo "   $an_issue\n";
                }
            }

            if ($nb_errors_introduced > 0)
            {
                echo "\nNb of new ERRORS: $nb_errors_introduced:\n";
                foreach ($this->diffs['errors_introduced'] as $an_issue)
                {
                    echo "   $an_issue\n";
                }

            }

            if ($nb_failures_fixed > 0)
            {
                echo "\nNb of newly FIXED FAILURES: $nb_failures_fixed:\n";
                foreach ($this->diffs['failures_fixed'] as $an_issue)
                {
                    echo "   $an_issue\n";
                }

            }

            if ($nb_errors_fixed > 0)
            {
                echo "\nNb of newly FIXED ERRORS: $nb_errors_fixed:\n";
                foreach ($this->diffs['errors_fixed'] as $an_issue)
                {
                    echo "   $an_issue\n";
                }

            }
        }
        else
        {
            echo "\n\nNo differences with baseline run. All is \"normal\".\n\n";
        }

        echo "\n\n";

    }

    function logname_current()
    {
        return $this->logname_stem.".current.json";
    }

    function logname_baseline()
    {
        return $this->logname_stem.".baseline.json";
    }

    function ask_if_want_to_create_baseline()
    {
        $answer = $this->prompt_for(
            "There is no baseline log. Would you like to log current failures and errors as the baseline?",
            array('y', 'n'));
        if ($answer == 'y')
        {
            $this->save_current_log_as_baseline();
        }
    }

    function process_phpunit_log_data($log_data)
    {
        global $tracer;

        $issues =
            array(
                'errors' => array(),
                'failures' => array(),
                'pass' => array()
            );

        foreach ($log_data as $log_entry)
        {
            $event = '';
            if (isset($log_entry['event']))
            {
                $event = $log_entry['event'];
            }

            if ($event != 'testStart' && $event != 'test')
            {
                continue;
            }

            $test = '';
            if (isset($log_entry['test']))
            {
                $test = $log_entry['test'];
            }

            if ($event == 'testStart')
            {
                $this->last_test_started = $test;
                continue;
            }
            else
            {
                $this->last_test_started = null;
            }

            /* For some reason, sometimes an event=test entry does not have
               a 'status' field.
               Whenever that happens, it seems to be a sign of an error.
            */
            $status = 'fail';
            if(isset($log_entry['status']))
            {
                $status = $log_entry['status'];
            }

            if ($status == 'fail')
            {
                array_push($issues['failures'], $test);
            }
            else if ($status == 'error')
            {
                array_push($issues['errors'], $test);
            }
            else if ($status == 'pass')
            {
                array_push($issues['pass'], $test);
            }
        }

        /* If a test was started by never ended, flag it as a failure */
        if ($this->last_test_started != null)
        {
            if (!in_array($this->last_test_started, $issues['failures'])) {
                array_push($issues['failures'], $this->last_test_started);
            }

        return $issues;

    }

    function compare_two_test_runs($baseline_issues, $current_issues)
    {
        global $tracer;

        $diffs = array('failures_introduced' => array(), 'failures_fixed' => array(),
            'errors_introduced' => array(), 'errors_fixed' => array());

        $current_failures = $current_issues['failures'];
        $current_pass = $current_issues['pass'];
        $baseline_failures = $baseline_issues['failures'];
        foreach ($baseline_failures as $a_baseline_failure)
        {
            if (in_array($a_baseline_failure, $current_pass))
            {
                array_push($diffs['failures_fixed'], $a_baseline_failure);
            }
        }

        foreach ($current_failures as $a_current_failure)
        {
            if (!in_array($a_current_failure, $baseline_failures))
            {
                array_push($diffs['failures_introduced'], $a_current_failure);<commit_msg>[Fix] Misc bugs in phpunit_with_baseline.php script and its supporting classes.<commit_after><?php
/**

 * Created by JetBrains PhpStorm.
 * User: alain_desilets
 * Date: 2013-10-02
 * Time: 2:12 PM
 * To change this template use File | Settings | File Templates.
 *
 * This class is used to run phpunit tests and compare the list of failures
 * and errors to those of a benchmark "normal" run.
 *
 * Use this class in situations where it's not practical for everyone to
 * keep all the tests "in the green" at all time, and to only commit code
 * that doesn't break any tests.
 *
 * With this class, you can tell if you have broken tests that were working
 * previously, or if you have fixed tests that were broken before.
 */

require_once('lib/debug/Tracer.php');

class TestRunnerWithBaseline {

    private $last_test_started = null;

    private $logname_stem = 'phpunit-log';
    public $action = 'run'; // run|update_baseline
    public $phpunit_options = '';
    public $help = false;
    public $diffs;

    function run()
    {
        global $tracer;

        $this->config_from_cmdline_options();

        if ($this->help) {
            $this->usage();
        }

        $this->run_tests();

        $this->print_diffs_with_baseline();

        if ($this->action == 'update_baseline')
        {
            $this->save_current_log_as_baseline();
        }

    }

    function run_tests()
    {
        global $tracer;

        $cmd_line = "../../vendor/bin/phpunit";
        if ($this->phpunit_options != '')
        {
            $cmd_line = "$cmd_line ".$this->phpunit_options;
        }

        $cmd_line = $cmd_line." --log-json \"".$this->logname_current()."\" .";

        print "\n=======\n === Executing phpunit as: '$cmd_line'\n=======\n\n";

        system($cmd_line);
    }

    function print_diffs_with_baseline()
    {
        global $tracer;

        echo "\n\nChecking for differences with baseline test logs...\n\n";

        $baseline_issues;
        if (file_exists($this->logname_baseline()))
        {
            $baseline_issues = $this->read_log_file($this->logname_baseline());
        }
        else
        {
            echo "=== WARNING: No baseline file exists. Assuming empty baseline.\n\n";
            $baseline_issues = $this->make_empty_issues_list();
        }

        $current_issues = $this->read_log_file($this->logname_current());

        $this->diffs = $this->compare_two_test_runs($baseline_issues, $current_issues);

        $nb_failures_introduced = count($this->diffs['failures_introduced']);
        $nb_failures_fixed = count($this->diffs['failures_fixed']);
        $nb_errors_introduced = count($this->diffs['errors_introduced']);
        $nb_errors_fixed = count($this->diffs['errors_fixed']);

        $total_diffs =
            $nb_failures_introduced + $nb_errors_introduced +
                $nb_failures_fixed + $nb_errors_fixed;

        if ($total_diffs > 0)
        {
            echo "\n\nThere were $total_diffs differences with baseline.\n

Below is a list of tests that differ from the baseline.
See above details about each error or failure.
";
            if ($nb_failures_introduced > 0)
            {
                echo "\nNb of new FAILURES: $nb_failures_introduced:\n";
                foreach ($this->diffs['failures_introduced'] as $an_issue)
                {
                    echo "   $an_issue\n";
                }
            }

            if ($nb_errors_introduced > 0)
            {
                echo "\nNb of new ERRORS: $nb_errors_introduced:\n";
                foreach ($this->diffs['errors_introduced'] as $an_issue)
                {
                    echo "   $an_issue\n";
                }

            }

            if ($nb_failures_fixed > 0)
            {
                echo "\nNb of newly FIXED FAILURES: $nb_failures_fixed:\n";
                foreach ($this->diffs['failures_fixed'] as $an_issue)
                {
                    echo "   $an_issue\n";
                }

            }

            if ($nb_errors_fixed > 0)
            {
                echo "\nNb of newly FIXED ERRORS: $nb_errors_fixed:\n";
                foreach ($this->diffs['errors_fixed'] as $an_issue)
                {
                    echo "   $an_issue\n";
                }

            }
        }
        else
        {
            echo "\n\nNo differences with baseline run. All is \"normal\".\n\n";
        }

        echo "\n\n";

    }

    function logname_current()
    {
        return $this->logname_stem.".current.json";
    }

    function logname_baseline()
    {
        return $this->logname_stem.".baseline.json";
    }

    function ask_if_want_to_create_baseline()
    {
        $answer = $this->prompt_for(
            "There is no baseline log. Would you like to log current failures and errors as the baseline?",
            array('y', 'n'));
        if ($answer == 'y')
        {
            $this->save_current_log_as_baseline();
        }
    }

    function process_phpunit_log_data($log_data)
    {
        global $tracer;

        $issues =
            array(
                'errors' => array(),
                'failures' => array(),
                'pass' => array()
            );

        foreach ($log_data as $log_entry)
        {
            $event = '';
            if (isset($log_entry['event']))
            {
                $event = $log_entry['event'];
            }

            if ($event != 'testStart' && $event != 'test')
            {
                continue;
            }

            $test = '';
            if (isset($log_entry['test']))
            {
                $test = $log_entry['test'];
            }

            if ($event == 'testStart')
            {
                $this->last_test_started = $test;
                continue;
            }
            else
            {
                $this->last_test_started = null;
            }

            /* For some reason, sometimes an event=test entry does not have
               a 'status' field.
               Whenever that happens, it seems to be a sign of an error.
            */
            $status = 'fail';
            if(isset($log_entry['status']))
            {
                $status = $log_entry['status'];
            }

            if ($status == 'fail')
            {
                array_push($issues['failures'], $test);
            }
            else if ($status == 'error')
            {
                array_push($issues['errors'], $test);
            }
            else if ($status == 'pass')
            {
                array_push($issues['pass'], $test);
            }
        }

        /* If a test was started by never ended, flag it as a failure */
        if ($this->last_test_started != null)
        {
            if (!in_array($this->last_test_started, $issues['failures'])) {
                array_push($issues['failures'], $this->last_test_started);
            }
        }

        return $issues;

    }

    function compare_two_test_runs($baseline_issues, $current_issues)
    {
        global $tracer;

        $diffs = array('failures_introduced' => array(), 'failures_fixed' => array(),
            'errors_introduced' => array(), 'errors_fixed' => array());

        $current_failures = $current_issues['failures'];
        $current_pass = $current_issues['pass'];
        $baseline_failures = $baseline_issues['failures'];
        foreach ($baseline_failures as $a_baseline_failure)
        {
            if (in_array($a_baseline_failure, $current_pass))
            {
                array_push($diffs['failures_fixed'], $a_baseline_failure);
            }
        }

        foreach ($current_failures as $a_current_failure)
        {
            if (!in_array($a_current_failure, $baseline_failures))
            {
                array_push($diffs['failures_introduced'], $a_current_failure);
{'fraction_non_alphanumeric': 0.12544688457609807, 'fraction_numerical': 0.0020429009193054137, 'mean_word_length': 1.66547558278033, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '347574', 'n_tokens_mistral': 4691, 'n_tokens_neox': 4365, 'n_words': 990}