<commit_before>#!/usr/bin/env python
# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run layout tests using the test_shell.

This is a port of the existing webkit test script run-webkit-tests.

The TestRunner class runs a series of tests (TestType interface) against a set
of test files.  If a test file fails a TestType, it returns a list TestFailure
objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
create a final report.

This script reads several files, if they exist in the test_lists subdirectory
next to this script itself.  Each should contain a list of paths to individual
tests or entire subdirectories of tests, relative to the outermost test
directory.  Entire lines starting with '//' (comments) will be ignored.

For details of the files' contents and purposes, see test_lists/README.
"""

import errno
import glob
import logging
import math
import optparse
import os
import Queue
import random
import re
import shutil
import subprocess
import sys
import time
import traceback

from layout_package import compare_failures
from layout_package import test_expectations
from layout_package import http_server
from layout_package import json_results_generator
from layout_package import path_utils
from layout_package import test_failures
from layout_package import test_shell_thread
from test_types import fuzzy_image_diff
from test_types import image_diff
from test_types import test_type_base
from test_types import text_diff
from test_types import simplified_text_diff

class TestInfo:
  """Groups information about a test for easy passing of data."""
  def __init__(self, filename, timeout):
    """Generates the URI and stores the filename and timeout for this test.
    Args:
      filename: Full path to the test.
      timeout: Timeout for running the test in TestShell.
      """
    self.filename = filename
    self.uri = path_utils.FilenameToUri(filename)
    self.timeout = timeout
    expected_hash_file = path_utils.ExpectedFilename(filename, '.checksum')
    try:
      self.image_hash = open(expected_hash_file, "r").read()
    except IOError, e:
      if errno.ENOENT != e.errno:
        raise
      self.image_hash = None


class ResultSummaryEntry:
  def __init__(self, all, failed, failure_counts, skipped):
    """Resolves result counts.

    Args:
      all: list of all tests in this category
      failed: list of failing tests in this category
      skipped: list of skipped tests
      failure_counts: dictionary of (TestFailure -> frequency)
    """
    self.skip_count = len(skipped)
    self.total_count = len(all | skipped)
    self.pass_count = self.total_count - self.skip_count - len(failed)
    self.failure_counts = failure_counts;


class ResultSummary:
  """Container for all result summaries for this test run.

  Args:
    deferred: ResultSummary object for deferred tests.
    wontfix: ResultSummary object for wontfix tests.
    fixable: ResultSummary object for tests marked in test_expectations.txt as
        needing fixing but are not deferred/wontfix (i.e. should be fixed
        for the next release).
    fixable_count: Count of all fixable and skipped tests. This is essentially
        a deduped sum of all the failures that are not deferred/wontfix.
    all_fixable_count: Count of all the tests that are not deferred/wontfix.
        This includes passing tests.
  """
  def __init__(self, deferred, wontfix, fixable, fixable_count,
      all_fixable_count):
    self.deferred = deferred
    self.wontfix = wontfix
    self.fixable = fixable
    self.fixable_count = fixable_count
    self.all_fixable_count = all_fixable_count


class TestRunner:
  """A class for managing running a series of tests on a series of test
  files."""

  # When collecting test cases, we include any file with these extensions.
  _supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl',
                                    '.php', '.svg'])
  # When collecting test cases, skip these directories
  _skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests'])

  # Top-level directories to shard when running tests.
  _shardable_directories = set(['chrome', 'LayoutTests', 'pending'])

  HTTP_SUBDIR = os.sep.join(['', 'http', ''])

  # The per-test timeout in milliseconds, if no --time-out-ms option was given
  # to run_webkit_tests. This should correspond to the default timeout in
  # test_shell.exe.
  DEFAULT_TEST_TIMEOUT_MS = 10 * 1000

  def __init__(self, options, paths):
    """Collect a list of files to test.

    Args:
      options: a dictionary of command line options
      paths: a list of paths to crawl looking for test files
    """
    self._options = options

    self._http_server = http_server.Lighttpd(options.results_directory)
    # a list of TestType objects
    self._test_types = []

    # a set of test files, and the same tests as a list
    self._test_files = set()
    self._test_files_list = None
    self._file_dir = path_utils.GetAbsolutePath(os.path.dirname(sys.argv[0]))

    if options.lint_test_files:
      # Creating the expecations for each platform/target pair does all the
      # test list parsing and ensures it's correct syntax (e.g. no dupes).
      for platform in test_expectations.TestExpectationsFile.PLATFORMS:
        self._ParseExpectations(platform, is_debug_mode=True)
        self._ParseExpectations(platform, is_debug_mode=False)
    else:
      self._GatherTestFiles(paths)
      self._expectations = self._ParseExpectations(options.platform,
          options.target == 'Debug')
      self._PrepareListsAndPrintOutput()

  def __del__(self):
    logging.info("flushing stdout")
    sys.stdout.flush()
    logging.info("flushing stderr")
    sys.stderr.flush()
    logging.info("stopping http server")
    # Stop the http server.
    self._http_server.Stop()

  def _GatherTestFiles(self, paths):
    """Generate a set of test files and place them in self._test_files

    Args:
      paths: a list of command line paths relative to the webkit/tests
             directory.  glob patterns are ok.
    """
    paths_to_walk = set()
    # if paths is empty, provide a pre-defined list.
    if not paths:
        paths = TestRunner._shardable_directories
    for path in paths:
      # If there's an * in the name, assume it's a glob pattern.
      path = os.path.join(path_utils.LayoutTestsDir(path), path)
      if path.find('*') > -1:
        filenames = glob.glob(path)
        paths_to_walk.update(filenames)
      else:
        paths_to_walk.add(path)

    # Now walk all the paths passed in on the command line and get filenames
    for path in paths_to_walk:
      if os.path.isfile(path) and self._HasSupportedExtension(path):
        self._test_files.add(os.path.normpath(path))
        continue

      for root, dirs, files in os.walk(path):
        # don't walk skipped directories and sub directories
        if os.path.basename(root) in TestRunner._skipped_directories:
          del dirs[:]
          continue

        for filename in files:
          if self._HasSupportedExtension(filename):
            filename = os.path.join(root, filename)
            filename = os.path.normpath(filename)
            self._test_files.add(filename)

    logging.info('Found: %d tests' % len(self._test_files))

  def _ParseExpectations(self, platform, is_debug_mode):
    """Parse the expectations from the test_list files and return a data
    structure holding them. Throws an error if the test_list files have invalid
    syntax.
    """
    if self._options.lint_test_files:
      test_files = None
    else:
      test_files = self._test_files

    try:
      return test_expectations.TestExpectations(test_files,
                                                self._file_dir,
                                                platform,
                                                is_debug_mode)
    except Exception, err:
      if self._options.lint_test_files:
        print str(err)
      else:
        raise err

  def _PrepareListsAndPrintOutput(self):
    """Create appropriate subsets of test lists and print test counts.

    Create appropriate subsets of self._tests_files in
    self._ignored_failures, self._fixable_failures, and self._fixable_crashes.
    Also remove skipped files from self._test_files, extract a subset of tests
    if desired, and create the sorted self._test_files_list.
    """
    # Remove skipped - both fixable and ignored - files from the
    # top-level list of files to test.
    skipped = set()
    # If there was only one test file, we'll run it even if it was skipped.
    if len(self._test_files) > 1 and not self._options.force:
      skipped = (self._expectations.GetFixableSkipped() |
                 self._expectations.GetDeferredSkipped() |
                 self._expectations.GetWontFixSkipped())
      self._test_files -= skipped

    if self._options.force:
      logging.info('Skipped: 0 tests (--force)')
    else:
      logging.info('Skipped: %d tests' % len(skipped))
    logging.info('Skipped tests do not appear in any of the below numbers\n')

    # Create a sorted list of test files so the subset chunk, if used, contains
    # alphabetically consecutive tests.
    self._test_files_list = list(self._test_files)
    if self._options.randomize_order:
      random.shuffle(self._test_files_list)
    else:
      self._test_files_list.sort(self.TestFilesSort)

    # If the user specifies they just want to run a subset of the tests,
    # just grab a subset of the non-skipped tests.
    if self._options.run_chunk or self._options.run_part:
      chunk_value = self._options.run_chunk or self._options.run_part
      test_files = self._test_files_list
      try:
        (chunk_num, chunk_len) = chunk_value.split(":")
        chunk_num = int(chunk_num)
        assert(chunk_num >= 0)
        test_size = int(chunk_len)
        assert(test_size > 0)
      except:
        logging.critical("invalid chunk '%s'" % chunk_value)
        sys.exit(1)

      # Get the number of tests
      num_tests = len(test_files)

      # Get the start offset of the slice.
      if self._options.run_chunk:
        chunk_len = test_size
        # In this case chunk_num can be really large. We need to make the
        # slave fit in the current number of tests.
        slice_start = (chunk_num * chunk_len) % num_tests
      else:
        # Validate the data.
        assert(test_size <= num_tests)
        assert(chunk_num <= test_size)

        # To count the chunk_len, and make sure we don't skip some tests, we
        # round to the next value that fits exacly all the parts.
        rounded_tests = num_tests
        if rounded_tests % test_size != 0:
          rounded_tests = num_tests + test_size - (num_tests % test_size)

        chunk_len = rounded_tests / test_size
        slice_start = chunk_len * (chunk_num - 1)
        # It does not mind if we go over test_size.

      # Get the end offset of the slice.
      slice_end = min(num_tests, slice_start + chunk_len)

      files = test_files[slice_start:slice_end]
      tests_run_msg = 'Run: %d tests (chunk slice [%d:%d] of %d)' % (
          (slice_end - slice_start), slice_start, slice_end, num_tests)
      logging.info(tests_run_msg)

      # If we reached the end and we don't have enough tests, we run some
      # from the beginning.
      if self._options.run_chunk and (slice_end - slice_start < chunk_len):
        extra = 1 + chunk_len - (slice_end - slice_start)
        extra_msg = '   last chunk is partial, appending [0:%d]' % extra
        logging.info(extra_msg)
        tests_run_msg += "\n" + extra_msg
        files.extend(test_files[0:extra])
      self._test_files_list = files
      self._test_files = set(files)

      tests_run_filename = os.path.join(self._options.results_directory,
                                        "tests_run.txt")
      tests_run_file = open(tests_run_filename, "w")
      tests_run_file.write(tests_run_msg + "\n")
      tests_run_file.close()

      # update expectations so that the stats are calculated correctly
      self._expectations = self._ParseExpectations(
          path_utils.PlatformName(), options.target == 'Debug')
    else:
      logging.info('Run: %d tests' % len(self._test_files))

    logging.info('Deferred: %d tests' % len(self._expectations.GetDeferred()))
    logging.info('Expected passes: %d tests' %
                 len(self._test_files -
                     self._expectations.GetFixable() -
                     self._expectations.GetWontFix()))
    logging.info(('Expected failures: %d fixable, %d ignored '
                  'and %d deferred tests') %
                 (len(self._expectations.GetFixableFailures()),
                  len(self._expectations.GetWontFixFailures()),
                  len(self._expectations.GetDeferredFailures())))
    logging.info(('Expected timeouts: %d fixable, %d ignored '
                  'and %d deferred tests') %
                 (len(self._expectations.GetFixableTimeouts()),
                  len(self._expectations.GetWontFixTimeouts()),
                  len(self._expectations.GetDeferredTimeouts())))
    logging.info('Expected crashes: %d fixable tests' %
                 len(self._expectations.GetFixableCrashes()))

  def _HasSupportedExtension(self, filename):
    """Return true if filename is one of the file extensions we want to run a
    test on."""
    extension = os.path.splitext(filename)[1]
    return extension in TestRunner._supported_file_extensions

  def AddTestType(self, test_type):
    """Add a TestType to the TestRunner."""
    self._test_types.append(test_type)

  # We sort the tests so that tests using the http server will run first.  We
  # are seeing some flakiness, maybe related to apache getting swapped out,
  # slow, or stuck after not serving requests for a while.
  def TestFilesSort(self, x, y):
    """Sort with http tests always first."""
    x_is_http = x.find(self.HTTP_SUBDIR) >= 0
    y_is_http = y.find(self.HTTP_SUBDIR) >= 0
    if x_is_http != y_is_http:
      return cmp(y_is_http, x_is_http)
    return cmp(x, y)

  def _GetDirForTestFile(self, test_file):
    """Returns the highest-level directory by which to shard the given test
    file."""
    # TODO(ojan): See if we can grab the lowest level directory. That will
    # provide better parallelization. We should at least be able to do so
    # for some directories (e.g. LayoutTests/dom).
    index = test_file.rfind(os.sep + 'LayoutTests' + os.sep)
    if index is -1:
      index = test_file.rfind(os.sep + 'chrome' + os.sep)

    test_file = test_file[index + 1:]
    test_file_parts = test_file.split(os.sep, 1)
    directory = test_file_parts[0]
    test_file = test_file_parts[1]

    return_value = directory
    while directory in TestRunner._shardable_directories:
      test_file_parts = test_file.split(os.sep, 1)
      directory = test_file_parts[0]
      return_value = os.path.join(return_value, directory)
      test_file = test_file_parts[1]

    return return_value

  def _GetTestFileQueue(self, test_files):
    """Create the thread safe queue of lists of (test filenames, test URIs)
    tuples. Each TestShellThread pulls a list from this queue and runs those
    tests in order before grabbing the next available list.

    Shard the lists by directory. This helps ensure that tests that depend
    on each other (aka bad tests!) continue to run together as most
    cross-tests dependencies tend to occur within the same directory.

    Return:
      The Queue of lists of TestInfo objects.
    """
    tests_by_dir = {}
    for test_file in test_files:
      directory = self._GetDirForTestFile(test_file)
      if directory not in tests_by_dir:
        tests_by_dir[directory] = []

      if self._expectations.HasModifier(test_file, test_expectations.SLOW):
        timeout = str(10 * int(options.time_out_ms))
      else:
        timeout = self._options.time_out_ms

      tests_by_dir[directory].append(TestInfo(test_file, timeout))

    # Sort by the number of tests in the dir so that the ones with the most
    # tests get run first in order to maximize parallelization. Number of tests
    # is a good enough, but not perfect, approximation of how long that set of
    # tests will take to run. We can't just use a PriorityQueue until we move
    # to Python 2.6.
    test_lists = []
    http_tests = None
    for directory in tests_by_dir:
      test_list = tests_by_dir[directory]
      # Keep the tests in alphabetical order.
      # TODO: Remove once tests are fixed so they can be run in any order.
      test_list.reverse()
      test_list_tuple = (directory, test_list)
      if directory == 'LayoutTests' + os.sep + 'http':
        http_tests = test_list_tuple
      else:
        test_lists.append(test_list_tuple)
    test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))

    # Put the http tests first. There are only a couple hundred of them, but
    # each http test takes a very long time to run, so sorting by the number
    # of tests doesn't accurately capture how long they take to run.
    if http_tests:
      test_lists.insert(0, http_tests)

    filename_queue = Queue.Queue()
    for item in test_lists:
      filename_queue.put(item)
    return filename_queue

  def _GetTestShellArgs(self, index):
    """Returns the tuple of arguments for tests and for test_shell."""
    shell_args = []
    test_args = test_type_base.TestArguments()
    if not self._options.no_pixel_tests:
      png_path = os.path.join(self._options.results_directory,
                              "png_result%s.png" % index)
      shell_args.append("--pixel-tests=" + png_path)
      test_args.png_path = png_path

    test_args.new_baseline = self._options.new_baseline
    test_args.show_sources = self._options.sources

    if self._options.startup_dialog:
      shell_args.append('--testshell-startup-dialog')

    if self._options.gp_fault_error_box:
      shell_args.append('--gp-fault-error-box')

    return (test_args, shell_args)

  def _InstantiateTestShellThreads(self, test_shell_binary):
    """Instantitates and starts the TestShellThread(s).

    Return:
      The list of threads.
    """
    test_shell_command = [test_shell_binary]

    if self._options.wrapper:
      # This split() isn't really what we want -- it incorrectly will
      # split quoted strings within the wrapper argument -- but in
      # practice it shouldn't come up and the --help output warns
      # about it anyway.
      test_shell_command = self._options.wrapper.split() + test_shell_command

    test_files = self._test_files_list
    filename_queue = self._GetTestFileQueue(test_files)

    # If we have http tests, the first one will be an http test.
    if test_files and test_files[0].find(self.HTTP_SUBDIR) >= 0:
      self._http_server.Start()

    # Instantiate TestShellThreads and start them.
    threads = []
    for i in xrange(int(self._options.num_test_shells)):
      # Create separate TestTypes instances for each thread.
      test_types = []
      for t in self._test_types:
        test_types.append(t(self._options.platform,
                            self._options.results_directory))

      test_args, shell_args = self._GetTestShellArgs(i)
      thread = test_shell_thread.TestShellThread(filename_queue,
                                                 test_shell_command,
                                                 test_types,
                                                 test_args,
                                                 shell_args,
                                                 self._options)
      thread.start()
      threads.append(thread)

    return threads

  def _StopLayoutTestHelper(self, proc):
    """Stop the layout test helper and closes it down."""
    if proc:
      logging.info("Stopping layout test helper")
      proc.stdin.write("x\n")
      proc.stdin.close()
      proc.wait()

  def Run(self):
    """Run all our tests on all our test files.

    For each test file, we run each test type.  If there are any failures, we
    collect them for reporting.

    Return:
      We return nonzero if there are regressions compared to the last run.
    """
    if not self._test_files:
      return 0
    start_time = time.time()
    test_shell_binary = path_utils.TestShellPath(self._options.target)

    # Start up any helper needed
    layout_test_helper_proc = None
    if not options.no_pixel_tests:
      helper_path = path_utils.LayoutTestHelperPath(self._options.target)
      if len(helper_path):
        logging.info("Starting layout helper %s" % helper_path)
        layout_test_helper_proc = subprocess.Popen([helper_path],
                                                   stdin=subprocess.PIPE,
                                                   stdout=subprocess.PIPE,
                                                   stderr=None)
        is_ready = layout_test_helper_proc.stdout.readline()
        if not is_ready.startswith('ready'):
          logging.error("layout_test_helper failed to be ready")

    # Check that the system dependencies (themes, fonts, ...) are correct.
    if not self._options.nocheck_sys_deps:
      proc = subprocess.Popen([test_shell_binary,
                              "--check-layout-test-sys-deps"])
      if proc.wait() != 0:
        logging.info("Aborting because system dependencies check failed.\n"
                     "To override, invoke with --nocheck-sys-deps")
        sys.exit(1)

    logging.info("Starting tests")

    threads = self._InstantiateTestShellThreads(test_shell_binary)

    # Wait for the threads to finish and collect test failures.
    failures = {}
    test_timings = {}
    individual_test_timings = []
    try:
      for thread in threads:
        while thread.isAlive():
          # Let it timeout occasionally so it can notice a KeyboardInterrupt
          # Actually, the timeout doesn't really matter: apparently it
          # suffices to not use an indefinite blocking join for it to
          # be interruptible by KeyboardInterrupt.
          thread.join(1.0)
        failures.update(thread.GetFailures())
        test_timings.update(thread.GetDirectoryTimingStats())
        individual_test_timings.extend(thread.GetIndividualTestStats())
    except KeyboardInterrupt:
      for thread in threads:
        thread.Cancel()
      self._StopLayoutTestHelper(layout_test_helper_proc)
      raise
    self._StopLayoutTestHelper(layout_test_helper_proc)
    for thread in threads:
      # Check whether a TestShellThread died before normal completion.
      exception_info = thread.GetExceptionInfo()
      if exception_info is not None:
        # Re-raise the thread's exception here to make it clear that
        # testing was aborted. Otherwise, the tests that did not run
        # would be assumed to have passed.
        raise exception_info[0], exception_info[1], exception_info[2]

    print
    end_time = time.time()
    logging.info("%f total testing time" % (end_time - start_time))

    print
    self._PrintTimingStatistics(test_timings, individual_test_timings, failures)

    print "-" * 78

    # Tests are done running. Compare failures with expected failures.
    regressions = self._CompareFailures(failures)

    print "-" * 78

    # Write summaries to stdout.
    result_summary = self._GetResultSummary(failures)
    self._PrintResultSummary(result_summary, sys.stdout)

    if self._options.verbose:
      self._WriteJSONFiles(failures, individual_test_timings, result_summary);

    # Write the same data to a log file.
    out_filename = os.path.join(self._options.results_directory, "score.txt")
    output_file = open(out_filename, "w")
    self._PrintResultSummary(result_summary, output_file)
    output_file.close()

    # Write the summary to disk (results.html) and maybe open the test_shell
    # to this file.
    wrote_results = self._WriteResultsHtmlFile(failures, regressions)
    if not self._options.noshow_results and wrote_results:
      self._ShowResultsHtmlFile()

    sys.stdout.flush()
    sys.stderr.flush()
    return len(regressions)

  def _WriteJSONFiles(self, failures, individual_test_timings, result_summary):
    logging.debug("Writing JSON files in %s." % self._options.results_directory)
    # Write a json file of the test_expectations.txt file for the layout tests
    # dashboard.
    expectations_file = open(os.path.join(self._options.results_directory,
        "expectations.json"), "w")
    expectations_json = self._expectations.GetExpectationsJsonForAllPlatforms()
    expectations_file.write(("ADD_EXPECTATIONS(" + expectations_json + ");"))
    expectations_file.close()

    results_file_path = os.path.join(self._options.results_directory,
        "results.json")
    builder_name = self._options.builder_name # "WebKitBuilder"
    build_number = self._options.build_number # "12346"
    json_generator = json_results_generator.JSONResultsGenerator(failures,
        individual_test_timings, builder_name, build_number,
        results_file_path, self._test_files_list, result_summary)
    results_json = json_generator.GetJSON()

    results_file = open(results_file_path, "w")
    results_file.write(results_json)
    results_file.close()

    logging.debug("Finished writing JSON files.")

  def _PrintTimingStatistics(self, directory_test_timings,
      individual_test_timings, failures):
    self._PrintAggregateTestStatistics(individual_test_timings)
    self._PrintIndividualTestTimes(individual_test_timings, failures)
    self._PrintDirectoryTimings(directory_test_timings)

  def _PrintAggregateTestStatistics(self, individual_test_timings):
    """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
    Args:
      individual_test_timings: List of test_shell_thread.TestStats for all
          tests.
    """
    test_types = individual_test_timings[0].time_for_diffs.keys()
    times_for_test_shell = []
    times_for_diff_processing = []
    times_per_test_type = {}
    for test_type in test_types:
      times_per_test_type[test_type] = []

    for test_stats in individual_test_timings:
      times_for_test_shell.append(test_stats.test_run_time)
      times_for_diff_processing.append(test_stats.total_time_for_all_diffs)
      time_for_diffs = test_stats.time_for_diffs
      for test_type in test_types:
        times_per_test_type[test_type].append(time_for_diffs[test_type])

    self._PrintStatisticsForTestTimings(
        "PER TEST TIME IN TESTSHELL (seconds):",
        times_for_test_shell)
    self._PrintStatisticsForTestTimings(
        "PER TEST DIFF PROCESSING TIMES (seconds):",
        times_for_diff_processing)
    for test_type in test_types:
      self._PrintStatisticsForTestTimings(
          "PER TEST TIMES BY TEST TYPE: %s" % test_type,
          times_per_test_type[test_type])

  def _PrintIndividualTestTimes(self, individual_test_timings, failures):
    """Prints the run times for slow, timeout and crash tests.
    Args:
      individual_test_timings: List of test_shell_thread.TestStats for all
          tests.
      failures: Dictionary mapping test filenames to list of test_failures.
    """
    # Reverse-sort by the time spent in test_shell.
    individual_test_timings.sort(lambda a, b:
        cmp(b.test_run_time, a.test_run_time))

    num_printed = 0
    slow_tests = []
    timeout_or_crash_tests = []
    unexpected_slow_tests = []
    for test_tuple in individual_test_timings:
      filename = test_tuple.filename
      is_timeout_crash_or_slow = False
      if self._expectations.HasModifier(filename, test_expectations.SLOW):
        is_timeout_crash_or_slow = True
        slow_tests.append(test_tuple)

      if filename in failures:
        for failure in failures[filename]:
          if (failure.__class__ == test_failures.FailureTimeout or
              failure.__class__ == test_failures.FailureCrash):
            is_timeout_crash_or_slow = True
            timeout_or_crash_tests.append(test_tuple)
            break

      if (not is_timeout_crash_or_slow and
          num_printed < self._options.num_slow_tests_to_log):
        num_printed = num_printed + 1
        unexpected_slow_tests.append(test_tuple)

    print
    self._PrintTestListTiming("%s slowest tests that are not marked as SLOW "
        "and did not timeout/crash:" % self._options.num_slow_tests_to_log,
        unexpected_slow_tests)
    print
    self._PrintTestListTiming("Tests marked as SLOW:", slow_tests)
    print
    self._PrintTestListTiming("Tests that timed out or crashed:",
        timeout_or_crash_tests)
    print

  def _PrintTestListTiming(self, title, test_list):
    logging.debug(title)
    for test_tuple in test_list:
      filename = test_tuple.filename[len(path_utils.LayoutTestsDir()) + 1:]
      filename = filename.replace('\\', '/')
      test_run_time = round(test_tuple.test_run_time, 1)
      logging.debug("%s took %s seconds" % (filename, test_run_time))

  def _PrintDirectoryTimings(self, directory_test_timings):
    timings = []
    for directory in directory_test_timings:
      num_tests, time_for_directory = directory_test_timings[directory]
      timings.append((round(time_for_directory, 1), directory, num_tests))
    timings.sort()

    logging.debug("Time to process each subdirectory:")
    for timing in timings:
      logging.debug("%s took %s seconds to run %s tests." % \
                    (timing[1], timing[0], timing[2]))

  def _PrintStatisticsForTestTimings(self, title, timings):
    """Prints the median, mean and standard deviation of the values in timings.
    Args:
      title: Title for these timings.
      timings: A list of floats representing times.
    """
    logging.debug(title)
    timings.sort()

    num_tests = len(timings)
    percentile90 = timings[int(.9 * num_tests)]
    percentile99 = timings[int(.99 * num_tests)]

    if num_tests % 2 == 1:
      median = timings[((num_tests - 1) / 2) - 1]
    else:
      lower = timings[num_tests / 2 - 1]
      upper = timings[num_tests / 2]
      median = (float(lower + upper)) / 2

    mean = sum(timings) / num_tests

    for time in timings:
      sum_of_deviations = math.pow(time - mean, 2)

    std_deviation = math.sqrt(sum_of_deviations / num_tests)
    logging.debug(("Median: %s, Mean: %s, 90th percentile: %s, "
        "99th percentile: %s, Standard deviation: %s\n" % (
        median, mean, percentile90, percentile99, std_deviation)))

  def _GetResultSummary(self, failures):
    """Returns a ResultSummary object with failure counts.

    Args:
      failures: dictionary mapping the test filename to a list of
          TestFailure objects if the test failed
    """
    fixable_counts = {}
    deferred_counts = {}
    wontfix_counts = {}

    fixable_failures = set()
    deferred_failures = set()
    wontfix_failures = set()

    # Aggregate failures in a dictionary (TestFailure -> frequency),
    # with known (fixable and wontfix) failures separated out.
    def AddFailure(dictionary, key):
      if key in dictionary:
        dictionary[key] += 1
      else:
        dictionary[key] = 1

    for test, failures in failures.iteritems():
      for failure in failures:
        if self._expectations.IsDeferred(test):
          count_group = deferred_counts
          failure_group = deferred_failures
        elif self._expectations.IsIgnored(test):
          count_group = wontfix_counts
          failure_group = wontfix_failures
        else:
          count_group = fixable_counts
          failure_group = fixable_failures

        AddFailure(count_group, failure.__class__)
        failure_group.add(test)

    deduped_fixable_count = len(fixable_failures |
        self._expectations.GetFixableSkipped())
    all_fixable_count = len(self._test_files -
        self._expectations.GetWontFix() - self._expectations.GetDeferred())

    # Breakdown of tests we need to fix and want to pass.
    # Include skipped fixable tests in the statistics.
    fixable_result_summary = ResultSummaryEntry(
        self._expectations.GetFixable() | fixable_failures,
        fixable_failures,
        fixable_counts,
        self._expectations.GetFixableSkipped())

    deferred_result_summary = ResultSummaryEntry(
        self._expectations.GetDeferred(),
        deferred_failures,
        deferred_counts,
        self._expectations.GetDeferredSkipped())

    wontfix_result_summary = ResultSummaryEntry(
        self._expectations.GetWontFix(),
        wontfix_failures,
        wontfix_counts,
        self._expectations.GetWontFixSkipped())

    return ResultSummary(deferred_result_summary, wontfix_result_summary,
        fixable_result_summary, deduped_fixable_count, all_fixable_count)

  def _PrintResultSummary(self, result_summary, output):
    """Print a short summary to stdout about how many tests passed.

    Args:
      result_summary: ResultSummary object with failure counts.
      output: file descriptor to write the results to. For example, sys.stdout.
    """
    failed = result_summary.fixable_count
    total = result_summary.all_fixable_count
    output.write(
        "\nTest summary: %.1f%% Passed | %s Failures | %s Tests to pass for "
        "this release\n" % (<commit_msg>Include skipped tests statistics when running sharded tests. Review URL: http://codereview.chromium.org/215054<commit_after>#!/usr/bin/env python
# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run layout tests using the test_shell.

This is a port of the existing webkit test script run-webkit-tests.

The TestRunner class runs a series of tests (TestType interface) against a set
of test files.  If a test file fails a TestType, it returns a list TestFailure
objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
create a final report.

This script reads several files, if they exist in the test_lists subdirectory
next to this script itself.  Each should contain a list of paths to individual
tests or entire subdirectories of tests, relative to the outermost test
directory.  Entire lines starting with '//' (comments) will be ignored.

For details of the files' contents and purposes, see test_lists/README.
"""

import errno
import glob
import logging
import math
import optparse
import os
import Queue
import random
import re
import shutil
import subprocess
import sys
import time
import traceback

from layout_package import compare_failures
from layout_package import test_expectations
from layout_package import http_server
from layout_package import json_results_generator
from layout_package import path_utils
from layout_package import test_failures
from layout_package import test_shell_thread
from test_types import fuzzy_image_diff
from test_types import image_diff
from test_types import test_type_base
from test_types import text_diff
from test_types import simplified_text_diff

class TestInfo:
  """Groups information about a test for easy passing of data."""
  def __init__(self, filename, timeout):
    """Generates the URI and stores the filename and timeout for this test.
    Args:
      filename: Full path to the test.
      timeout: Timeout for running the test in TestShell.
      """
    self.filename = filename
    self.uri = path_utils.FilenameToUri(filename)
    self.timeout = timeout
    expected_hash_file = path_utils.ExpectedFilename(filename, '.checksum')
    try:
      self.image_hash = open(expected_hash_file, "r").read()
    except IOError, e:
      if errno.ENOENT != e.errno:
        raise
      self.image_hash = None


class ResultSummaryEntry:
  def __init__(self, all, failed, failure_counts, skipped):
    """Resolves result counts.

    Args:
      all: list of all tests in this category
      failed: list of failing tests in this category
      skipped: list of skipped tests
      failure_counts: dictionary of (TestFailure -> frequency)
    """
    self.skip_count = len(skipped)
    self.total_count = len(all | skipped)
    self.pass_count = self.total_count - self.skip_count - len(failed)
    self.failure_counts = failure_counts;


class ResultSummary:
  """Container for all result summaries for this test run.

  Args:
    deferred: ResultSummary object for deferred tests.
    wontfix: ResultSummary object for wontfix tests.
    fixable: ResultSummary object for tests marked in test_expectations.txt as
        needing fixing but are not deferred/wontfix (i.e. should be fixed
        for the next release).
    fixable_count: Count of all fixable and skipped tests. This is essentially
        a deduped sum of all the failures that are not deferred/wontfix.
    all_fixable_count: Count of all the tests that are not deferred/wontfix.
        This includes passing tests.
  """
  def __init__(self, deferred, wontfix, fixable, fixable_count,
      all_fixable_count):
    self.deferred = deferred
    self.wontfix = wontfix
    self.fixable = fixable
    self.fixable_count = fixable_count
    self.all_fixable_count = all_fixable_count


class TestRunner:
  """A class for managing running a series of tests on a series of test
  files."""

  # When collecting test cases, we include any file with these extensions.
  _supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl',
                                    '.php', '.svg'])
  # When collecting test cases, skip these directories
  _skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests'])

  # Top-level directories to shard when running tests.
  _shardable_directories = set(['chrome', 'LayoutTests', 'pending'])

  HTTP_SUBDIR = os.sep.join(['', 'http', ''])

  # The per-test timeout in milliseconds, if no --time-out-ms option was given
  # to run_webkit_tests. This should correspond to the default timeout in
  # test_shell.exe.
  DEFAULT_TEST_TIMEOUT_MS = 10 * 1000

  def __init__(self, options, paths):
    """Collect a list of files to test.

    Args:
      options: a dictionary of command line options
      paths: a list of paths to crawl looking for test files
    """
    self._options = options

    self._http_server = http_server.Lighttpd(options.results_directory)
    # a list of TestType objects
    self._test_types = []

    # a set of test files, and the same tests as a list
    self._test_files = set()
    self._test_files_list = None
    self._file_dir = path_utils.GetAbsolutePath(os.path.dirname(sys.argv[0]))

    if options.lint_test_files:
      # Creating the expecations for each platform/target pair does all the
      # test list parsing and ensures it's correct syntax (e.g. no dupes).
      for platform in test_expectations.TestExpectationsFile.PLATFORMS:
        self._ParseExpectations(platform, is_debug_mode=True)
        self._ParseExpectations(platform, is_debug_mode=False)
    else:
      self._GatherTestFiles(paths)
      self._expectations = self._ParseExpectations(options.platform,
          options.target == 'Debug')
      self._PrepareListsAndPrintOutput()

  def __del__(self):
    logging.info("flushing stdout")
    sys.stdout.flush()
    logging.info("flushing stderr")
    sys.stderr.flush()
    logging.info("stopping http server")
    # Stop the http server.
    self._http_server.Stop()

  def _GatherTestFiles(self, paths):
    """Generate a set of test files and place them in self._test_files

    Args:
      paths: a list of command line paths relative to the webkit/tests
             directory.  glob patterns are ok.
    """
    paths_to_walk = set()
    # if paths is empty, provide a pre-defined list.
    if not paths:
        paths = TestRunner._shardable_directories
    for path in paths:
      # If there's an * in the name, assume it's a glob pattern.
      path = os.path.join(path_utils.LayoutTestsDir(path), path)
      if path.find('*') > -1:
        filenames = glob.glob(path)
        paths_to_walk.update(filenames)
      else:
        paths_to_walk.add(path)

    # Now walk all the paths passed in on the command line and get filenames
    for path in paths_to_walk:
      if os.path.isfile(path) and self._HasSupportedExtension(path):
        self._test_files.add(os.path.normpath(path))
        continue

      for root, dirs, files in os.walk(path):
        # don't walk skipped directories and sub directories
        if os.path.basename(root) in TestRunner._skipped_directories:
          del dirs[:]
          continue

        for filename in files:
          if self._HasSupportedExtension(filename):
            filename = os.path.join(root, filename)
            filename = os.path.normpath(filename)
            self._test_files.add(filename)

    logging.info('Found: %d tests' % len(self._test_files))

  def _ParseExpectations(self, platform, is_debug_mode):
    """Parse the expectations from the test_list files and return a data
    structure holding them. Throws an error if the test_list files have invalid
    syntax.
    """
    if self._options.lint_test_files:
      test_files = None
    else:
      test_files = self._test_files

    try:
      return test_expectations.TestExpectations(test_files,
                                                self._file_dir,
                                                platform,
                                                is_debug_mode)
    except Exception, err:
      if self._options.lint_test_files:
        print str(err)
      else:
        raise err

  def _PrepareListsAndPrintOutput(self):
    """Create appropriate subsets of test lists and print test counts.

    Create appropriate subsets of self._tests_files in
    self._ignored_failures, self._fixable_failures, and self._fixable_crashes.
    Also remove skipped files from self._test_files, extract a subset of tests
    if desired, and create the sorted self._test_files_list.
    """
    # Remove skipped - both fixable and ignored - files from the
    # top-level list of files to test.
    skipped = set()
    # If there was only one test file, we'll run it even if it was skipped.
    if len(self._test_files) > 1 and not self._options.force:
      skipped = (self._expectations.GetFixableSkipped() |
                 self._expectations.GetDeferredSkipped() |
                 self._expectations.GetWontFixSkipped())
      self._test_files -= skipped

    if self._options.force:
      logging.info('Skipped: 0 tests (--force)')
    else:
      logging.info('Skipped: %d tests' % len(skipped))
    logging.info('Skipped tests do not appear in any of the below numbers\n')

    # Create a sorted list of test files so the subset chunk, if used, contains
    # alphabetically consecutive tests.
    self._test_files_list = list(self._test_files)
    if self._options.randomize_order:
      random.shuffle(self._test_files_list)
    else:
      self._test_files_list.sort(self.TestFilesSort)

    # Chunking replaces self._expectations, which loses all the skipped test
    # information. Keep the prechunk expectations for tracking number of
    # skipped tests.
    self.prechunk_expectations = self._expectations;

    # If the user specifies they just want to run a subset of the tests,
    # just grab a subset of the non-skipped tests.
    if self._options.run_chunk or self._options.run_part:
      chunk_value = self._options.run_chunk or self._options.run_part
      test_files = self._test_files_list
      try:
        (chunk_num, chunk_len) = chunk_value.split(":")
        chunk_num = int(chunk_num)
        assert(chunk_num >= 0)
        test_size = int(chunk_len)
        assert(test_size > 0)
      except:
        logging.critical("invalid chunk '%s'" % chunk_value)
        sys.exit(1)

      # Get the number of tests
      num_tests = len(test_files)

      # Get the start offset of the slice.
      if self._options.run_chunk:
        chunk_len = test_size
        # In this case chunk_num can be really large. We need to make the
        # slave fit in the current number of tests.
        slice_start = (chunk_num * chunk_len) % num_tests
      else:
        # Validate the data.
        assert(test_size <= num_tests)
        assert(chunk_num <= test_size)

        # To count the chunk_len, and make sure we don't skip some tests, we
        # round to the next value that fits exacly all the parts.
        rounded_tests = num_tests
        if rounded_tests % test_size != 0:
          rounded_tests = num_tests + test_size - (num_tests % test_size)

        chunk_len = rounded_tests / test_size
        slice_start = chunk_len * (chunk_num - 1)
        # It does not mind if we go over test_size.

      # Get the end offset of the slice.
      slice_end = min(num_tests, slice_start + chunk_len)

      files = test_files[slice_start:slice_end]
      tests_run_msg = 'Run: %d tests (chunk slice [%d:%d] of %d)' % (
          (slice_end - slice_start), slice_start, slice_end, num_tests)
      logging.info(tests_run_msg)

      # If we reached the end and we don't have enough tests, we run some
      # from the beginning.
      if self._options.run_chunk and (slice_end - slice_start < chunk_len):
        extra = 1 + chunk_len - (slice_end - slice_start)
        extra_msg = '   last chunk is partial, appending [0:%d]' % extra
        logging.info(extra_msg)
        tests_run_msg += "\n" + extra_msg
        files.extend(test_files[0:extra])
      self._test_files_list = files
      self._test_files = set(files)

      tests_run_filename = os.path.join(self._options.results_directory,
                                        "tests_run.txt")
      tests_run_file = open(tests_run_filename, "w")
      tests_run_file.write(tests_run_msg + "\n")
      tests_run_file.close()

      # update expectations so that the stats are calculated correctly
      self._expectations = self._ParseExpectations(
          path_utils.PlatformName(), options.target == 'Debug')
    else:
      logging.info('Run: %d tests' % len(self._test_files))

    logging.info('Deferred: %d tests' % len(self._expectations.GetDeferred()))
    logging.info('Expected passes: %d tests' %
                 len(self._test_files -
                     self._expectations.GetFixable() -
                     self._expectations.GetWontFix()))
    logging.info(('Expected failures: %d fixable, %d ignored '
                  'and %d deferred tests') %
                 (len(self._expectations.GetFixableFailures()),
                  len(self._expectations.GetWontFixFailures()),
                  len(self._expectations.GetDeferredFailures())))
    logging.info(('Expected timeouts: %d fixable, %d ignored '
                  'and %d deferred tests') %
                 (len(self._expectations.GetFixableTimeouts()),
                  len(self._expectations.GetWontFixTimeouts()),
                  len(self._expectations.GetDeferredTimeouts())))
    logging.info('Expected crashes: %d fixable tests' %
                 len(self._expectations.GetFixableCrashes()))

  def _HasSupportedExtension(self, filename):
    """Return true if filename is one of the file extensions we want to run a
    test on."""
    extension = os.path.splitext(filename)[1]
    return extension in TestRunner._supported_file_extensions

  def AddTestType(self, test_type):
    """Add a TestType to the TestRunner."""
    self._test_types.append(test_type)

  # We sort the tests so that tests using the http server will run first.  We
  # are seeing some flakiness, maybe related to apache getting swapped out,
  # slow, or stuck after not serving requests for a while.
  def TestFilesSort(self, x, y):
    """Sort with http tests always first."""
    x_is_http = x.find(self.HTTP_SUBDIR) >= 0
    y_is_http = y.find(self.HTTP_SUBDIR) >= 0
    if x_is_http != y_is_http:
      return cmp(y_is_http, x_is_http)
    return cmp(x, y)

  def _GetDirForTestFile(self, test_file):
    """Returns the highest-level directory by which to shard the given test
    file."""
    # TODO(ojan): See if we can grab the lowest level directory. That will
    # provide better parallelization. We should at least be able to do so
    # for some directories (e.g. LayoutTests/dom).
    index = test_file.rfind(os.sep + 'LayoutTests' + os.sep)
    if index is -1:
      index = test_file.rfind(os.sep + 'chrome' + os.sep)

    test_file = test_file[index + 1:]
    test_file_parts = test_file.split(os.sep, 1)
    directory = test_file_parts[0]
    test_file = test_file_parts[1]

    return_value = directory
    while directory in TestRunner._shardable_directories:
      test_file_parts = test_file.split(os.sep, 1)
      directory = test_file_parts[0]
      return_value = os.path.join(return_value, directory)
      test_file = test_file_parts[1]

    return return_value

  def _GetTestFileQueue(self, test_files):
    """Create the thread safe queue of lists of (test filenames, test URIs)
    tuples. Each TestShellThread pulls a list from this queue and runs those
    tests in order before grabbing the next available list.

    Shard the lists by directory. This helps ensure that tests that depend
    on each other (aka bad tests!) continue to run together as most
    cross-tests dependencies tend to occur within the same directory.

    Return:
      The Queue of lists of TestInfo objects.
    """
    tests_by_dir = {}
    for test_file in test_files:
      directory = self._GetDirForTestFile(test_file)
      if directory not in tests_by_dir:
        tests_by_dir[directory] = []

      if self._expectations.HasModifier(test_file, test_expectations.SLOW):
        timeout = str(10 * int(options.time_out_ms))
      else:
        timeout = self._options.time_out_ms

      tests_by_dir[directory].append(TestInfo(test_file, timeout))

    # Sort by the number of tests in the dir so that the ones with the most
    # tests get run first in order to maximize parallelization. Number of tests
    # is a good enough, but not perfect, approximation of how long that set of
    # tests will take to run. We can't just use a PriorityQueue until we move
    # to Python 2.6.
    test_lists = []
    http_tests = None
    for directory in tests_by_dir:
      test_list = tests_by_dir[directory]
      # Keep the tests in alphabetical order.
      # TODO: Remove once tests are fixed so they can be run in any order.
      test_list.reverse()
      test_list_tuple = (directory, test_list)
      if directory == 'LayoutTests' + os.sep + 'http':
        http_tests = test_list_tuple
      else:
        test_lists.append(test_list_tuple)
    test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))

    # Put the http tests first. There are only a couple hundred of them, but
    # each http test takes a very long time to run, so sorting by the number
    # of tests doesn't accurately capture how long they take to run.
    if http_tests:
      test_lists.insert(0, http_tests)

    filename_queue = Queue.Queue()
    for item in test_lists:
      filename_queue.put(item)
    return filename_queue

  def _GetTestShellArgs(self, index):
    """Returns the tuple of arguments for tests and for test_shell."""
    shell_args = []
    test_args = test_type_base.TestArguments()
    if not self._options.no_pixel_tests:
      png_path = os.path.join(self._options.results_directory,
                              "png_result%s.png" % index)
      shell_args.append("--pixel-tests=" + png_path)
      test_args.png_path = png_path

    test_args.new_baseline = self._options.new_baseline
    test_args.show_sources = self._options.sources

    if self._options.startup_dialog:
      shell_args.append('--testshell-startup-dialog')

    if self._options.gp_fault_error_box:
      shell_args.append('--gp-fault-error-box')

    return (test_args, shell_args)

  def _InstantiateTestShellThreads(self, test_shell_binary):
    """Instantitates and starts the TestShellThread(s).

    Return:
      The list of threads.
    """
    test_shell_command = [test_shell_binary]

    if self._options.wrapper:
      # This split() isn't really what we want -- it incorrectly will
      # split quoted strings within the wrapper argument -- but in
      # practice it shouldn't come up and the --help output warns
      # about it anyway.
      test_shell_command = self._options.wrapper.split() + test_shell_command

    test_files = self._test_files_list
    filename_queue = self._GetTestFileQueue(test_files)

    # If we have http tests, the first one will be an http test.
    if test_files and test_files[0].find(self.HTTP_SUBDIR) >= 0:
      self._http_server.Start()

    # Instantiate TestShellThreads and start them.
    threads = []
    for i in xrange(int(self._options.num_test_shells)):
      # Create separate TestTypes instances for each thread.
      test_types = []
      for t in self._test_types:
        test_types.append(t(self._options.platform,
                            self._options.results_directory))

      test_args, shell_args = self._GetTestShellArgs(i)
      thread = test_shell_thread.TestShellThread(filename_queue,
                                                 test_shell_command,
                                                 test_types,
                                                 test_args,
                                                 shell_args,
                                                 self._options)
      thread.start()
      threads.append(thread)

    return threads

  def _StopLayoutTestHelper(self, proc):
    """Stop the layout test helper and closes it down."""
    if proc:
      logging.info("Stopping layout test helper")
      proc.stdin.write("x\n")
      proc.stdin.close()
      proc.wait()

  def Run(self):
    """Run all our tests on all our test files.

    For each test file, we run each test type.  If there are any failures, we
    collect them for reporting.

    Return:
      We return nonzero if there are regressions compared to the last run.
    """
    if not self._test_files:
      return 0
    start_time = time.time()
    test_shell_binary = path_utils.TestShellPath(self._options.target)

    # Start up any helper needed
    layout_test_helper_proc = None
    if not options.no_pixel_tests:
      helper_path = path_utils.LayoutTestHelperPath(self._options.target)
      if len(helper_path):
        logging.info("Starting layout helper %s" % helper_path)
        layout_test_helper_proc = subprocess.Popen([helper_path],
                                                   stdin=subprocess.PIPE,
                                                   stdout=subprocess.PIPE,
                                                   stderr=None)
        is_ready = layout_test_helper_proc.stdout.readline()
        if not is_ready.startswith('ready'):
          logging.error("layout_test_helper failed to be ready")

    # Check that the system dependencies (themes, fonts, ...) are correct.
    if not self._options.nocheck_sys_deps:
      proc = subprocess.Popen([test_shell_binary,
                              "--check-layout-test-sys-deps"])
      if proc.wait() != 0:
        logging.info("Aborting because system dependencies check failed.\n"
                     "To override, invoke with --nocheck-sys-deps")
        sys.exit(1)

    logging.info("Starting tests")

    threads = self._InstantiateTestShellThreads(test_shell_binary)

    # Wait for the threads to finish and collect test failures.
    failures = {}
    test_timings = {}
    individual_test_timings = []
    try:
      for thread in threads:
        while thread.isAlive():
          # Let it timeout occasionally so it can notice a KeyboardInterrupt
          # Actually, the timeout doesn't really matter: apparently it
          # suffices to not use an indefinite blocking join for it to
          # be interruptible by KeyboardInterrupt.
          thread.join(1.0)
        failures.update(thread.GetFailures())
        test_timings.update(thread.GetDirectoryTimingStats())
        individual_test_timings.extend(thread.GetIndividualTestStats())
    except KeyboardInterrupt:
      for thread in threads:
        thread.Cancel()
      self._StopLayoutTestHelper(layout_test_helper_proc)
      raise
    self._StopLayoutTestHelper(layout_test_helper_proc)
    for thread in threads:
      # Check whether a TestShellThread died before normal completion.
      exception_info = thread.GetExceptionInfo()
      if exception_info is not None:
        # Re-raise the thread's exception here to make it clear that
        # testing was aborted. Otherwise, the tests that did not run
        # would be assumed to have passed.
        raise exception_info[0], exception_info[1], exception_info[2]

    print
    end_time = time.time()
    logging.info("%f total testing time" % (end_time - start_time))

    print
    self._PrintTimingStatistics(test_timings, individual_test_timings, failures)

    print "-" * 78

    # Tests are done running. Compare failures with expected failures.
    regressions = self._CompareFailures(failures)

    print "-" * 78

    # Write summaries to stdout.
    result_summary = self._GetResultSummary(failures)
    self._PrintResultSummary(result_summary, sys.stdout)

    if self._options.verbose:
      self._WriteJSONFiles(failures, individual_test_timings, result_summary);

    # Write the same data to a log file.
    out_filename = os.path.join(self._options.results_directory, "score.txt")
    output_file = open(out_filename, "w")
    self._PrintResultSummary(result_summary, output_file)
    output_file.close()

    # Write the summary to disk (results.html) and maybe open the test_shell
    # to this file.
    wrote_results = self._WriteResultsHtmlFile(failures, regressions)
    if not self._options.noshow_results and wrote_results:
      self._ShowResultsHtmlFile()

    sys.stdout.flush()
    sys.stderr.flush()
    return len(regressions)

  def _WriteJSONFiles(self, failures, individual_test_timings, result_summary):
    logging.debug("Writing JSON files in %s." % self._options.results_directory)
    # Write a json file of the test_expectations.txt file for the layout tests
    # dashboard.
    expectations_file = open(os.path.join(self._options.results_directory,
        "expectations.json"), "w")
    expectations_json = self._expectations.GetExpectationsJsonForAllPlatforms()
    expectations_file.write(("ADD_EXPECTATIONS(" + expectations_json + ");"))
    expectations_file.close()

    results_file_path = os.path.join(self._options.results_directory,
        "results.json")
    builder_name = self._options.builder_name # "WebKitBuilder"
    build_number = self._options.build_number # "12346"
    json_generator = json_results_generator.JSONResultsGenerator(failures,
        individual_test_timings, builder_name, build_number,
        results_file_path, self._test_files_list, result_summary)
    results_json = json_generator.GetJSON()

    results_file = open(results_file_path, "w")
    results_file.write(results_json)
    results_file.close()

    logging.debug("Finished writing JSON files.")

  def _PrintTimingStatistics(self, directory_test_timings,
      individual_test_timings, failures):
    self._PrintAggregateTestStatistics(individual_test_timings)
    self._PrintIndividualTestTimes(individual_test_timings, failures)
    self._PrintDirectoryTimings(directory_test_timings)

  def _PrintAggregateTestStatistics(self, individual_test_timings):
    """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
    Args:
      individual_test_timings: List of test_shell_thread.TestStats for all
          tests.
    """
    test_types = individual_test_timings[0].time_for_diffs.keys()
    times_for_test_shell = []
    times_for_diff_processing = []
    times_per_test_type = {}
    for test_type in test_types:
      times_per_test_type[test_type] = []

    for test_stats in individual_test_timings:
      times_for_test_shell.append(test_stats.test_run_time)
      times_for_diff_processing.append(test_stats.total_time_for_all_diffs)
      time_for_diffs = test_stats.time_for_diffs
      for test_type in test_types:
        times_per_test_type[test_type].append(time_for_diffs[test_type])

    self._PrintStatisticsForTestTimings(
        "PER TEST TIME IN TESTSHELL (seconds):",
        times_for_test_shell)
    self._PrintStatisticsForTestTimings(
        "PER TEST DIFF PROCESSING TIMES (seconds):",
        times_for_diff_processing)
    for test_type in test_types:
      self._PrintStatisticsForTestTimings(
          "PER TEST TIMES BY TEST TYPE: %s" % test_type,
          times_per_test_type[test_type])

  def _PrintIndividualTestTimes(self, individual_test_timings, failures):
    """Prints the run times for slow, timeout and crash tests.
    Args:
      individual_test_timings: List of test_shell_thread.TestStats for all
          tests.
      failures: Dictionary mapping test filenames to list of test_failures.
    """
    # Reverse-sort by the time spent in test_shell.
    individual_test_timings.sort(lambda a, b:
        cmp(b.test_run_time, a.test_run_time))

    num_printed = 0
    slow_tests = []
    timeout_or_crash_tests = []
    unexpected_slow_tests = []
    for test_tuple in individual_test_timings:
      filename = test_tuple.filename
      is_timeout_crash_or_slow = False
      if self._expectations.HasModifier(filename, test_expectations.SLOW):
        is_timeout_crash_or_slow = True
        slow_tests.append(test_tuple)

      if filename in failures:
        for failure in failures[filename]:
          if (failure.__class__ == test_failures.FailureTimeout or
              failure.__class__ == test_failures.FailureCrash):
            is_timeout_crash_or_slow = True
            timeout_or_crash_tests.append(test_tuple)
            break

      if (not is_timeout_crash_or_slow and
          num_printed < self._options.num_slow_tests_to_log):
        num_printed = num_printed + 1
        unexpected_slow_tests.append(test_tuple)

    print
    self._PrintTestListTiming("%s slowest tests that are not marked as SLOW "
        "and did not timeout/crash:" % self._options.num_slow_tests_to_log,
        unexpected_slow_tests)
    print
    self._PrintTestListTiming("Tests marked as SLOW:", slow_tests)
    print
    self._PrintTestListTiming("Tests that timed out or crashed:",
        timeout_or_crash_tests)
    print

  def _PrintTestListTiming(self, title, test_list):
    logging.debug(title)
    for test_tuple in test_list:
      filename = test_tuple.filename[len(path_utils.LayoutTestsDir()) + 1:]
      filename = filename.replace('\\', '/')
      test_run_time = round(test_tuple.test_run_time, 1)
      logging.debug("%s took %s seconds" % (filename, test_run_time))

  def _PrintDirectoryTimings(self, directory_test_timings):
    timings = []
    for directory in directory_test_timings:
      num_tests, time_for_directory = directory_test_timings[directory]
      timings.append((round(time_for_directory, 1), directory, num_tests))
    timings.sort()

    logging.debug("Time to process each subdirectory:")
    for timing in timings:
      logging.debug("%s took %s seconds to run %s tests." % \
                    (timing[1], timing[0], timing[2]))

  def _PrintStatisticsForTestTimings(self, title, timings):
    """Prints the median, mean and standard deviation of the values in timings.
    Args:
      title: Title for these timings.
      timings: A list of floats representing times.
    """
    logging.debug(title)
    timings.sort()

    num_tests = len(timings)
    percentile90 = timings[int(.9 * num_tests)]
    percentile99 = timings[int(.99 * num_tests)]

    if num_tests % 2 == 1:
      median = timings[((num_tests - 1) / 2) - 1]
    else:
      lower = timings[num_tests / 2 - 1]
      upper = timings[num_tests / 2]
      median = (float(lower + upper)) / 2

    mean = sum(timings) / num_tests

    for time in timings:
      sum_of_deviations = math.pow(time - mean, 2)

    std_deviation = math.sqrt(sum_of_deviations / num_tests)
    logging.debug(("Median: %s, Mean: %s, 90th percentile: %s, "
        "99th percentile: %s, Standard deviation: %s\n" % (
        median, mean, percentile90, percentile99, std_deviation)))

  def _GetResultSummary(self, failures):
    """Returns a ResultSummary object with failure counts.

    Args:
      failures: dictionary mapping the test filename to a list of
          TestFailure objects if the test failed
    """
    fixable_counts = {}
    deferred_counts = {}
    wontfix_counts = {}

    fixable_failures = set()
    deferred_failures = set()
    wontfix_failures = set()

    # Aggregate failures in a dictionary (TestFailure -> frequency),
    # with known (fixable and wontfix) failures separated out.
    def AddFailure(dictionary, key):
      if key in dictionary:
        dictionary[key] += 1
      else:
        dictionary[key] = 1

    for test, failures in failures.iteritems():
      for failure in failures:
        if self._expectations.IsDeferred(test):
          count_group = deferred_counts
          failure_group = deferred_failures
        elif self._expectations.IsIgnored(test):
          count_group = wontfix_counts
          failure_group = wontfix_failures
        else:
          count_group = fixable_counts
          failure_group = fixable_failures

        AddFailure(count_group, failure.__class__)
        failure_group.add(test)

    # Here and below, use the prechuncked expectations object for numbers of
    # skipped tests. Chunking removes the skipped tests before creating the
    # expectations file.
    #
    # This is a bit inaccurate, since the number of skipped tests will be
    # duplicated across all shard, but it's the best we can reasonably do.

    deduped_fixable_count = len(fixable_failures |
        self.prechunk_expectations.GetFixableSkipped())
    all_fixable_count = len(self._test_files -
        self._expectations.GetWontFix() - self._expectations.GetDeferred())

    # Breakdown of tests we need to fix and want to pass.
    # Include skipped fixable tests in the statistics.
    fixable_result_summary = ResultSummaryEntry(
        self._expectations.GetFixable() | fixable_failures,
        fixable_failures,
        fixable_counts,
        self.prechunk_expectations.GetFixableSkipped())

    deferred_result_summary = ResultSummaryEntry(
        self._expectations.GetDeferred(),
        deferred_failures,
        deferred_counts,
        self.prechunk_expectations.GetDeferredSkipped())

    wontfix_result_summary = ResultSummaryEntry(
        self._expectations.GetWontFix(),
        wontfix_failures,
        wontfix_counts,
        self.prechunk_expectations.GetWontFixSkipped())

    return ResultSummary(deferred_result_summary, wontfix_result_summary,
        fixable_result_summary, deduped_fixable_count, all_fixable_count)

  def _PrintResultSummary(self, result_summary, output):
    """Print a short summary to stdout about how many tests passed.

    Args:
      result_summary: ResultSummary object with failure counts.
      output: file descriptor to write the results to. For example, sys.stdout.
    """
    failed = result_summary.fixable_count
    total = result_summary.all_fixable_count
    output.write(
        "\nTest summary: %.1f%% Passed | %s Failures | %s Tests to pass for "
        "this release\n" % (
{'fraction_non_alphanumeric': 0.08085637081687799, 'fraction_numerical': 0.003058467203135671, 'mean_word_length': 2.902149354035108, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2758149', 'n_tokens_mistral': 19799, 'n_tokens_neox': 19166, 'n_words': 5658}