<commit_before>#!/bin/env python
# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run layout tests using the test_shell.

This is a port of the existing webkit test script run-webkit-tests.

The TestRunner class runs a series of tests (TestType interface) against a set
of test files.  If a test file fails a TestType, it returns a list TestFailure
objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
create a final report.

This script reads several files, if they exist in the test_lists subdirectory
next to this script itself.  Each should contain a list of paths to individual
tests or entire subdirectories of tests, relative to the outermost test
directory.  Entire lines starting with '//' (comments) will be ignored.

For details of the files' contents and purposes, see test_lists/README.
"""

import glob
import logging
import optparse
import os
import Queue
import random
import shutil
import subprocess
import sys
import time

import google.path_utils

from layout_package import compare_failures
from layout_package import test_expectations
from layout_package import http_server
from layout_package import path_utils
from layout_package import platform_utils
from layout_package import test_failures
from layout_package import test_shell_thread
from test_types import fuzzy_image_diff
from test_types import image_diff
from test_types import test_type_base
from test_types import text_diff
from test_types import simplified_text_diff


# The test list files are found in this subdirectory, which must be a sibling
# to this script itself.
TEST_FILE_DIR = 'test_lists'


class TestRunner:
  """A class for managing running a series of tests on a series of test
  files."""

  # When collecting test cases, we include any file with these extensions.
  _supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl',
                                    '.php', '.svg'])
  # When collecting test cases, skip these directories
  _skipped_directories = set(['.svn', '_svn', 'resources'])

  HTTP_SUBDIR = os.sep.join(['', 'http', ''])

  def __init__(self, options, paths):
    """Collect a list of files to test.

    Args:
      options: a dictionary of command line options
      paths: a list of paths to crawl looking for test files
    """
    self._options = options

    self._http_server = http_server.Lighttpd(options.results_directory)
    # a list of TestType objects
    self._test_types = []

    # a set of test files
    self._test_files = set()
    self._file_dir = os.path.join(os.path.dirname(sys.argv[0]), TEST_FILE_DIR)
    self._file_dir = path_utils.GetAbsolutePath(self._file_dir)

    if options.lint_test_files:
      # Creating the expecations for each platform/target pair does all the
      # test list parsing and ensures it's correct syntax(e.g. no dupes).
      self._ParseExpectations('win', is_debug_mode=True)
      self._ParseExpectations('win', is_debug_mode=False)
      self._ParseExpectations('mac', is_debug_mode=True)
      self._ParseExpectations('mac', is_debug_mode=False)
      self._ParseExpectations('linux', is_debug_mode=True)
      self._ParseExpectations('linux', is_debug_mode=False)
    else:
      self._GatherTestFiles(paths)
      self._expectations = self._ParseExpectations(
          platform_utils.GetTestListPlatformName().lower(),
          options.target == 'Debug')
      self._GenerateExpecationsAndPrintOutput()

  def __del__(self):
    sys.stdout.flush()
    sys.stderr.flush()
    # Stop the http server.
    self._http_server.Stop()

  def _GatherTestFiles(self, paths):
    """Generate a set of test files and place them in self._test_files

    Args:
      paths: a list of command line paths relative to the webkit/tests
             directory.  glob patterns are ok.
    """
    paths_to_walk = set()
    for path in paths:
      # If there's an * in the name, assume it's a glob pattern.
      path = os.path.join(path_utils.LayoutDataDir(), path)
      if path.find('*') > -1:
        filenames = glob.glob(path)
        paths_to_walk.update(filenames)
      else:
        paths_to_walk.add(path)

    # Now walk all the paths passed in on the command line and get filenames
    for path in paths_to_walk:
      if os.path.isfile(path) and self._HasSupportedExtension(path):
        self._test_files.add(os.path.normpath(path))
        continue

      for root, dirs, files in os.walk(path):
        # don't walk skipped directories and sub directories
        if os.path.basename(root) in TestRunner._skipped_directories:
          del dirs[:]
          continue

        for filename in files:
          if self._HasSupportedExtension(filename):
            filename = os.path.join(root, filename)
            filename = os.path.normpath(filename)
            self._test_files.add(filename)

    logging.info('Found: %d tests' % len(self._test_files))

  def _ParseExpectations(self, platform, is_debug_mode):
    """Parse the expectations from the test_list files and return a data
    structure holding them. Throws an error if the test_list files have invalid
    syntax.
    """
    test_files = self._options.lint_test_files ? None : self._test_files
    try:
      return test_expectations.TestExpectations(test_files,
                                                self._file_dir,
                                                platform,
                                                is_debug_mode)
    except SyntaxError, err:
      if self._options.lint_test_files:
        print str(err)
      else:
        raise err

  def _GenerateExpecationsAndPrintOutput(self):
    """Create appropriate subsets of self._tests_files in 
    self._ignored_failures, self._fixable_failures, and self._fixable_crashes.
    """
    # Filter and sort out files from the skipped, ignored, and fixable file
    # lists.
    saved_test_files = set()
    if len(self._test_files) == 1:
      # If there's only one test file, we don't want to skip it, but we do want
      # to sort it.  So we save it to add back to the list later.
      saved_test_files = self._test_files

    # Remove skipped - both fixable and ignored - files from the
    # top-level list of files to test.
    skipped = (self._expectations.GetFixableSkipped() |<commit_msg>Fix syntax error. TBR to green tree. Review URL: http://codereview.chromium.org/14804<commit_after>#!/bin/env python
# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run layout tests using the test_shell.

This is a port of the existing webkit test script run-webkit-tests.

The TestRunner class runs a series of tests (TestType interface) against a set
of test files.  If a test file fails a TestType, it returns a list TestFailure
objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
create a final report.

This script reads several files, if they exist in the test_lists subdirectory
next to this script itself.  Each should contain a list of paths to individual
tests or entire subdirectories of tests, relative to the outermost test
directory.  Entire lines starting with '//' (comments) will be ignored.

For details of the files' contents and purposes, see test_lists/README.
"""

import glob
import logging
import optparse
import os
import Queue
import random
import shutil
import subprocess
import sys
import time

import google.path_utils

from layout_package import compare_failures
from layout_package import test_expectations
from layout_package import http_server
from layout_package import path_utils
from layout_package import platform_utils
from layout_package import test_failures
from layout_package import test_shell_thread
from test_types import fuzzy_image_diff
from test_types import image_diff
from test_types import test_type_base
from test_types import text_diff
from test_types import simplified_text_diff


# The test list files are found in this subdirectory, which must be a sibling
# to this script itself.
TEST_FILE_DIR = 'test_lists'


class TestRunner:
  """A class for managing running a series of tests on a series of test
  files."""

  # When collecting test cases, we include any file with these extensions.
  _supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl',
                                    '.php', '.svg'])
  # When collecting test cases, skip these directories
  _skipped_directories = set(['.svn', '_svn', 'resources'])

  HTTP_SUBDIR = os.sep.join(['', 'http', ''])

  def __init__(self, options, paths):
    """Collect a list of files to test.

    Args:
      options: a dictionary of command line options
      paths: a list of paths to crawl looking for test files
    """
    self._options = options

    self._http_server = http_server.Lighttpd(options.results_directory)
    # a list of TestType objects
    self._test_types = []

    # a set of test files
    self._test_files = set()
    self._file_dir = os.path.join(os.path.dirname(sys.argv[0]), TEST_FILE_DIR)
    self._file_dir = path_utils.GetAbsolutePath(self._file_dir)

    if options.lint_test_files:
      # Creating the expecations for each platform/target pair does all the
      # test list parsing and ensures it's correct syntax(e.g. no dupes).
      self._ParseExpectations('win', is_debug_mode=True)
      self._ParseExpectations('win', is_debug_mode=False)
      self._ParseExpectations('mac', is_debug_mode=True)
      self._ParseExpectations('mac', is_debug_mode=False)
      self._ParseExpectations('linux', is_debug_mode=True)
      self._ParseExpectations('linux', is_debug_mode=False)
    else:
      self._GatherTestFiles(paths)
      self._expectations = self._ParseExpectations(
          platform_utils.GetTestListPlatformName().lower(),
          options.target == 'Debug')
      self._GenerateExpecationsAndPrintOutput()

  def __del__(self):
    sys.stdout.flush()
    sys.stderr.flush()
    # Stop the http server.
    self._http_server.Stop()

  def _GatherTestFiles(self, paths):
    """Generate a set of test files and place them in self._test_files

    Args:
      paths: a list of command line paths relative to the webkit/tests
             directory.  glob patterns are ok.
    """
    paths_to_walk = set()
    for path in paths:
      # If there's an * in the name, assume it's a glob pattern.
      path = os.path.join(path_utils.LayoutDataDir(), path)
      if path.find('*') > -1:
        filenames = glob.glob(path)
        paths_to_walk.update(filenames)
      else:
        paths_to_walk.add(path)

    # Now walk all the paths passed in on the command line and get filenames
    for path in paths_to_walk:
      if os.path.isfile(path) and self._HasSupportedExtension(path):
        self._test_files.add(os.path.normpath(path))
        continue

      for root, dirs, files in os.walk(path):
        # don't walk skipped directories and sub directories
        if os.path.basename(root) in TestRunner._skipped_directories:
          del dirs[:]
          continue

        for filename in files:
          if self._HasSupportedExtension(filename):
            filename = os.path.join(root, filename)
            filename = os.path.normpath(filename)
            self._test_files.add(filename)

    logging.info('Found: %d tests' % len(self._test_files))

  def _ParseExpectations(self, platform, is_debug_mode):
    """Parse the expectations from the test_list files and return a data
    structure holding them. Throws an error if the test_list files have invalid
    syntax.
    """
    if self._options.lint_test_files:
      test_files = None
    else:
      test_files = self._test_files

    try:
      return test_expectations.TestExpectations(test_files,
                                                self._file_dir,
                                                platform,
                                                is_debug_mode)
    except SyntaxError, err:
      if self._options.lint_test_files:
        print str(err)
      else:
        raise err

  def _GenerateExpecationsAndPrintOutput(self):
    """Create appropriate subsets of self._tests_files in 
    self._ignored_failures, self._fixable_failures, and self._fixable_crashes.
    """
    # Filter and sort out files from the skipped, ignored, and fixable file
    # lists.
    saved_test_files = set()
    if len(self._test_files) == 1:
      # If there's only one test file, we don't want to skip it, but we do want
      # to sort it.  So we save it to add back to the list later.
      saved_test_files = self._test_files

    # Remove skipped - both fixable and ignored - files from the
    # top-level list of files to test.
    skipped = (self._expectations.GetFixableSkipped() |
{'fraction_non_alphanumeric': 0.07613583501318034, 'fraction_numerical': 0.002093347805861374, 'mean_word_length': 3.06139798488665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '925425', 'n_tokens_mistral': 3669, 'n_tokens_neox': 3560, 'n_words': 1282}