<commit_before># Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Module for running cbuildbot stages in the background."""

import collections
import contextlib
import errno
import functools
import logging
import multiprocessing
import os
try:
  import Queue
except ImportError:
  # Python-3 renamed to "queue".  We still use Queue to avoid collisions
  # with naming variables as "queue".  Maybe we'll transition at some point.
  # pylint: disable=F0401
  import queue as Queue
import signal
import sys
import tempfile
import time
import traceback

from chromite.buildbot import cbuildbot_results as results_lib
from chromite.lib import cros_build_lib
from chromite.lib import osutils
from chromite.lib import timeout_util

_BUFSIZE = 1024

logger = logging.getLogger(__name__)


def Manager():
  """Create a background process for managing interprocess communication.

  This manager wraps multiprocessing.Manager() and ensures that any sockets
  created during initialization are created in '/tmp' rather than in a custom
  temp directory. This is needed because TMPDIR might be really long, and
  named sockets are limited to 108 characters.

  Usage:
    with Manager() as manager:
      queue = manager.Queue()
      ...

  Returns:
    The return value of multiprocessing.Manager()
  """
  old_tempdir_value, old_tempdir_env = osutils.SetGlobalTempDir('/tmp')
  try:
    return multiprocessing.Manager()
  finally:
    osutils.SetGlobalTempDir(old_tempdir_value, old_tempdir_env)


class BackgroundFailure(results_lib.StepFailure):
  """Exception to show a step failed while running in a background process."""


class _BackgroundTask(multiprocessing.Process):
  """Run a task in the background.

  This task may be the 'Run' function from a buildbot stage or just a plain
  function. It will be run in the background. Output from this task is saved
  to a temporary file and is printed when the 'Wait' function is called.
  """

  # The time we give Python to startup and exit.
  STARTUP_TIMEOUT = 60 * 5
  EXIT_TIMEOUT = 60 * 10

  # The time we allow processes to be silent. This must be greater than the
  # hw_test_timeout set in cbuildbot_config.py, and less than the timeout set
  # by buildbot itself (typically, 150 minutes.)
  SILENT_TIMEOUT = 60 * 205

  # The amount by which we reduce the SILENT_TIMEOUT every time we launch
  # a subprocess. This helps ensure that children get a chance to enforce the
  # SILENT_TIMEOUT prior to the parents enforcing it.
  SILENT_TIMEOUT_STEP = 30
  MINIMUM_SILENT_TIMEOUT = 60 * 135

  # The time before terminating or killing a task.<commit_msg>Add a TODO to investigate the correctness of a comment on hwtest timeouts.<commit_after># Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Module for running cbuildbot stages in the background."""

import collections
import contextlib
import errno
import functools
import logging
import multiprocessing
import os
try:
  import Queue
except ImportError:
  # Python-3 renamed to "queue".  We still use Queue to avoid collisions
  # with naming variables as "queue".  Maybe we'll transition at some point.
  # pylint: disable=F0401
  import queue as Queue
import signal
import sys
import tempfile
import time
import traceback

from chromite.buildbot import cbuildbot_results as results_lib
from chromite.lib import cros_build_lib
from chromite.lib import osutils
from chromite.lib import timeout_util

_BUFSIZE = 1024

logger = logging.getLogger(__name__)


def Manager():
  """Create a background process for managing interprocess communication.

  This manager wraps multiprocessing.Manager() and ensures that any sockets
  created during initialization are created in '/tmp' rather than in a custom
  temp directory. This is needed because TMPDIR might be really long, and
  named sockets are limited to 108 characters.

  Usage:
    with Manager() as manager:
      queue = manager.Queue()
      ...

  Returns:
    The return value of multiprocessing.Manager()
  """
  old_tempdir_value, old_tempdir_env = osutils.SetGlobalTempDir('/tmp')
  try:
    return multiprocessing.Manager()
  finally:
    osutils.SetGlobalTempDir(old_tempdir_value, old_tempdir_env)


class BackgroundFailure(results_lib.StepFailure):
  """Exception to show a step failed while running in a background process."""


class _BackgroundTask(multiprocessing.Process):
  """Run a task in the background.

  This task may be the 'Run' function from a buildbot stage or just a plain
  function. It will be run in the background. Output from this task is saved
  to a temporary file and is printed when the 'Wait' function is called.
  """

  # The time we give Python to startup and exit.
  STARTUP_TIMEOUT = 60 * 5
  EXIT_TIMEOUT = 60 * 10

  # TODO: crbug.com/360063
  # The time we allow processes to be silent.
  SILENT_TIMEOUT = 60 * 205

  # The amount by which we reduce the SILENT_TIMEOUT every time we launch
  # a subprocess. This helps ensure that children get a chance to enforce the
  # SILENT_TIMEOUT prior to the parents enforcing it.
  SILENT_TIMEOUT_STEP = 30
  MINIMUM_SILENT_TIMEOUT = 60 * 135

  # The time before terminating or killing a task.
{'fraction_non_alphanumeric': 0.055916775032509754, 'fraction_numerical': 0.014675831320824818, 'mean_word_length': 4.074458058435439, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7508520', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1475, 'n_words': 683}