<commit_before># Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A Thread object for running the test shell and processing URLs from a
shared queue.

Each thread runs a separate instance of the test_shell binary and validates
the output.  When there are no more URLs to process in the shared queue, the
thread exits.
"""

import copy
import logging
import os
import Queue
import subprocess
import thread
import threading

import path_utils
import platform_utils
import test_failures

# The per-test timeout in milliseconds, if no --time-out-ms option was given to
# run_webkit_tests. This should correspond to the default timeout in
# test_shell.exe.
DEFAULT_TEST_TIMEOUT_MS = 10 * 1000

def ProcessOutput(proc, filename, test_uri, test_types, test_args):
  """Receives the output from a test_shell process, subjects it to a number
  of tests, and returns a list of failure types the test produced.

  Args:
    proc: an active test_shell process
    filename: path of the test file being run
    test_types: list of test types to subject the output to
    test_args: arguments to be passed to each test

  Returns: a list of failure objects for the test being processed
  """
  outlines = []
  failures = []
  crash_or_timeout = False

  # Some test args, such as the image hash, may be added or changed on a
  # test-by-test basis.
  local_test_args = copy.copy(test_args)

  line = proc.stdout.readline()
  while line.rstrip() != "#EOF":
    # Make sure we haven't crashed.
    if line == '' and proc.poll() is not None:
      failures.append(test_failures.FailureCrash())

      # This is hex code 0xc000001d, which is used for abrupt termination.
      # This happens if we hit ctrl+c from the prompt and we happen to
      # be waiting on the test_shell.
      if -1073741510 == proc.returncode:
        raise KeyboardInterrupt
      crash_or_timeout = True
      break

    # Don't include #URL lines in our output
    if line.startswith("#URL:"):
      url = line.rstrip()[5:]
      if url != test_uri:
        logging.fatal("Test got out of sync:\n|%s|\n|%s|" %
                      (url, test_uri))
        raise AssertionError("test out of sync")
    elif line.startswith("#MD5:"):
      local_test_args.hash = line.rstrip()[5:]
    elif line.startswith("#TEST_TIMED_OUT"):
      # Test timed out, but we still need to read until #EOF.
      crash_or_timeout = True
      failures.append(test_failures.FailureTimeout())
    else:
      outlines.append(line)
    line = proc.stdout.readline()

  # If we had any stderr output, append that. This is not ideal, but at least
  # it'll catch errors.
  line = proc.stderr.readline()
  while line.rstrip() != "#EOF":
    # TODO(pamg): We suppress this stderr message temporarily so we can run
    # the tests deterministically until someone has a chance to fix the
    # underlying problem.
    # See http://code.google.com/p/chromium/issues/detail?id=4285
    if (line != '' and
        not line.endswith('alias ISO-8859-8-I maps to ISO-8859-8-I already, '
                          'but someone is trying to make it map to '
                          'ISO-8859-8')):
      outlines.append(line)
    line = proc.stderr.readline()

  # Check the output and save the results.
  for test_type in test_types:
    new_failures = test_type.CompareOutput(filename, proc,
                                           ''.join(outlines),
                                           local_test_args)
    # Don't add any more failures if we already have a crash or timeout, so
    # we don't double-report those tests.
    if not crash_or_timeout:
      failures.extend(new_failures)<commit_msg>Temporarily disable collecting and appending stderr output, to see if it makes the webkit tests stop hanging.<commit_after># Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A Thread object for running the test shell and processing URLs from a
shared queue.

Each thread runs a separate instance of the test_shell binary and validates
the output.  When there are no more URLs to process in the shared queue, the
thread exits.
"""

import copy
import logging
import os
import Queue
import subprocess
import thread
import threading

import path_utils
import platform_utils
import test_failures

# The per-test timeout in milliseconds, if no --time-out-ms option was given to
# run_webkit_tests. This should correspond to the default timeout in
# test_shell.exe.
DEFAULT_TEST_TIMEOUT_MS = 10 * 1000

def ProcessOutput(proc, filename, test_uri, test_types, test_args):
  """Receives the output from a test_shell process, subjects it to a number
  of tests, and returns a list of failure types the test produced.

  Args:
    proc: an active test_shell process
    filename: path of the test file being run
    test_types: list of test types to subject the output to
    test_args: arguments to be passed to each test

  Returns: a list of failure objects for the test being processed
  """
  outlines = []
  failures = []
  crash_or_timeout = False

  # Some test args, such as the image hash, may be added or changed on a
  # test-by-test basis.
  local_test_args = copy.copy(test_args)

  line = proc.stdout.readline()
  while line.rstrip() != "#EOF":
    # Make sure we haven't crashed.
    if line == '' and proc.poll() is not None:
      failures.append(test_failures.FailureCrash())

      # This is hex code 0xc000001d, which is used for abrupt termination.
      # This happens if we hit ctrl+c from the prompt and we happen to
      # be waiting on the test_shell.
      if -1073741510 == proc.returncode:
        raise KeyboardInterrupt
      crash_or_timeout = True
      break

    # Don't include #URL lines in our output
    if line.startswith("#URL:"):
      url = line.rstrip()[5:]
      if url != test_uri:
        logging.fatal("Test got out of sync:\n|%s|\n|%s|" %
                      (url, test_uri))
        raise AssertionError("test out of sync")
    elif line.startswith("#MD5:"):
      local_test_args.hash = line.rstrip()[5:]
    elif line.startswith("#TEST_TIMED_OUT"):
      # Test timed out, but we still need to read until #EOF.
      crash_or_timeout = True
      failures.append(test_failures.FailureTimeout())
    else:
      outlines.append(line)
    line = proc.stdout.readline()

  # If we had any stderr output, append that. This is not ideal, but at least
  # it'll catch errors.
#  line = proc.stderr.readline()
#  while line.rstrip() != "#EOF":
#    # TODO(pamg): We suppress this stderr message temporarily so we can run
#    # the tests deterministically until someone has a chance to fix the
#    # underlying problem.
#    # See http://code.google.com/p/chromium/issues/detail?id=4285
#    if (line != '' and
#        not line.endswith('alias ISO-8859-8-I maps to ISO-8859-8-I already, '
#                          'but someone is trying to make it map to '
#                          'ISO-8859-8')):
#      outlines.append(line)
#    line = proc.stderr.readline()

  # Check the output and save the results.
  for test_type in test_types:
    new_failures = test_type.CompareOutput(filename, proc,
                                           ''.join(outlines),
                                           local_test_args)
    # Don't add any more failures if we already have a crash or timeout, so
    # we don't double-report those tests.
    if not crash_or_timeout:
      failures.extend(new_failures)
{'fraction_non_alphanumeric': 0.08204456094364351, 'fraction_numerical': 0.013892529488859764, 'mean_word_length': 2.8231462925851702, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '676905', 'n_tokens_mistral': 2279, 'n_tokens_neox': 2167, 'n_words': 875}