<commit_before># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Base class for all slave-side build steps. """

import config
import multiprocessing
import os
import shlex
import signal
import subprocess
import sys
import time
import traceback

from playback_dirs import LocalSkpPlaybackDirs
from playback_dirs import StorageSkpPlaybackDirs
from utils import misc
from utils import shell_utils


# Add important directories to the PYTHONPATH
buildbot_root = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                             os.pardir, os.pardir)
sys.path.append(os.path.join(buildbot_root, 'site_config'))


DEFAULT_TIMEOUT = 2400
DEFAULT_NO_OUTPUT_TIMEOUT = 3600
DEFAULT_NUM_CORES = 2


# multiprocessing.Value doesn't accept boolean types, so we have to use an int.
INT_TRUE = 1
INT_FALSE = 0
build_step_stdout_has_written = multiprocessing.Value('i', INT_FALSE)


# The canned acl to use while copying playback files to Google Storage.
PLAYBACK_CANNED_ACL = 'private'


class BuildStepWarning(Exception):
  pass


class BuildStepFailure(Exception):
  pass


class BuildStepTimeout(Exception):
  pass


class BuildStepLogger(object):
  """ Override stdout so that we can keep track of when anything has been
  logged.  This enables timeouts based on how long the process has gone without
  writing output.
  """
  def __init__(self):
    self.stdout = sys.stdout
    sys.stdout = self
    build_step_stdout_has_written.value = INT_FALSE

  def __del__(self):
    sys.stdout = self.stdout

  def fileno(self):
    return self.stdout.fileno()

  def write(self, data):
    build_step_stdout_has_written.value = INT_TRUE
    self.stdout.write(data)

  def flush(self):
    self.stdout.flush()


class DeviceDirs(object):
  def __init__(self, perf_data_dir, gm_dir, skp_dir, skp_perf_dir, skp_out_dir):
    self._perf_data_dir = perf_data_dir
    self._gm_dir = gm_dir
    self._skp_dir = skp_dir
    self._skp_perf_dir = skp_perf_dir
    self._skp_out_dir = skp_out_dir

  def GMDir(self):
    return  self._gm_dir

  def PerfDir(self):
    return self._perf_data_dir

  def SKPDir(self):
    return self._skp_dir

  def SKPPerfDir(self):
    return self._skp_perf_dir

  def SKPOutDir(self):
    return self._skp_out_dir


class BuildStep(multiprocessing.Process):
  def __init__(self, args, attempts=1, timeout=DEFAULT_TIMEOUT,
               no_output_timeout=DEFAULT_NO_OUTPUT_TIMEOUT):
    """ Constructs a BuildStep instance.
    
    args: dictionary containing arguments to this BuildStep.
    attempts: how many times to try this BuildStep before giving up.
    timeout: maximum time allowed for this BuildStep.
    no_output_timeout: maximum time allowed for this BuildStep to run without
        any output.
    """
    multiprocessing.Process.__init__(self)
    self._args = args
    self.timeout = timeout
    self.no_output_timeout = no_output_timeout

    self._configuration = args['configuration']
    self._gm_image_subdir = args['gm_image_subdir']
    self._builder_name = args['builder_name']
    self._target_platform = args['target_platform']
    self._revision = \
        None if args['revision'] == 'None' else int(args['revision'])
    self._got_revision = \
        None if args['got_revision'] == 'None' else int(args['got_revision'])
    self.attempts = attempts
    self._do_upload_results = (False if args['do_upload_results'] == 'None'
                               else args['do_upload_results'] == 'True')
    # Figure out where we are going to store images generated by GM.
    self._gm_actual_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'actual')
    self._gm_merge_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'merge')
    self._gm_expected_dir = os.path.join(os.pardir, 'gm-expected',<commit_msg>If revision is 'HEAD' use None (for commit-queue triggered trybot runs).<commit_after># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Base class for all slave-side build steps. """

import config
import multiprocessing
import os
import shlex
import signal
import subprocess
import sys
import time
import traceback

from playback_dirs import LocalSkpPlaybackDirs
from playback_dirs import StorageSkpPlaybackDirs
from utils import misc
from utils import shell_utils


# Add important directories to the PYTHONPATH
buildbot_root = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                             os.pardir, os.pardir)
sys.path.append(os.path.join(buildbot_root, 'site_config'))


DEFAULT_TIMEOUT = 2400
DEFAULT_NO_OUTPUT_TIMEOUT = 3600
DEFAULT_NUM_CORES = 2


# multiprocessing.Value doesn't accept boolean types, so we have to use an int.
INT_TRUE = 1
INT_FALSE = 0
build_step_stdout_has_written = multiprocessing.Value('i', INT_FALSE)


# The canned acl to use while copying playback files to Google Storage.
PLAYBACK_CANNED_ACL = 'private'


class BuildStepWarning(Exception):
  pass


class BuildStepFailure(Exception):
  pass


class BuildStepTimeout(Exception):
  pass


class BuildStepLogger(object):
  """ Override stdout so that we can keep track of when anything has been
  logged.  This enables timeouts based on how long the process has gone without
  writing output.
  """
  def __init__(self):
    self.stdout = sys.stdout
    sys.stdout = self
    build_step_stdout_has_written.value = INT_FALSE

  def __del__(self):
    sys.stdout = self.stdout

  def fileno(self):
    return self.stdout.fileno()

  def write(self, data):
    build_step_stdout_has_written.value = INT_TRUE
    self.stdout.write(data)

  def flush(self):
    self.stdout.flush()


class DeviceDirs(object):
  def __init__(self, perf_data_dir, gm_dir, skp_dir, skp_perf_dir, skp_out_dir):
    self._perf_data_dir = perf_data_dir
    self._gm_dir = gm_dir
    self._skp_dir = skp_dir
    self._skp_perf_dir = skp_perf_dir
    self._skp_out_dir = skp_out_dir

  def GMDir(self):
    return  self._gm_dir

  def PerfDir(self):
    return self._perf_data_dir

  def SKPDir(self):
    return self._skp_dir

  def SKPPerfDir(self):
    return self._skp_perf_dir

  def SKPOutDir(self):
    return self._skp_out_dir


class BuildStep(multiprocessing.Process):
  def __init__(self, args, attempts=1, timeout=DEFAULT_TIMEOUT,
               no_output_timeout=DEFAULT_NO_OUTPUT_TIMEOUT):
    """ Constructs a BuildStep instance.
    
    args: dictionary containing arguments to this BuildStep.
    attempts: how many times to try this BuildStep before giving up.
    timeout: maximum time allowed for this BuildStep.
    no_output_timeout: maximum time allowed for this BuildStep to run without
        any output.
    """
    multiprocessing.Process.__init__(self)
    self._args = args
    self.timeout = timeout
    self.no_output_timeout = no_output_timeout

    self._configuration = args['configuration']
    self._gm_image_subdir = args['gm_image_subdir']
    self._builder_name = args['builder_name']
    self._target_platform = args['target_platform']
    self._revision = \
        None if args['revision'] == 'None' or args['revision'] == 'HEAD' \
        else int(args['revision'])
    self._got_revision = \
        None if args['got_revision'] == 'None' else int(args['got_revision'])
    self.attempts = attempts
    self._do_upload_results = (False if args['do_upload_results'] == 'None'
                               else args['do_upload_results'] == 'True')
    # Figure out where we are going to store images generated by GM.
    self._gm_actual_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'actual')
    self._gm_merge_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'merge')
    self._gm_expected_dir = os.path.join(os.pardir, 'gm-expected',
{'fraction_non_alphanumeric': 0.08673015873015873, 'fraction_numerical': 0.004063492063492063, 'mean_word_length': 3.8053691275167787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6525498', 'n_tokens_mistral': 2585, 'n_tokens_neox': 2480, 'n_words': 686}