<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Set of utilities to add commands to a buildbot factory (BuildFactory).

All the utility functions to add steps to a build factory here are not
project-specific. See the other *_commands.py for project-specific commands.
"""

import json
import ntpath
import posixpath
import re

from buildbot.locks import SlaveLock
from buildbot.process.properties import WithProperties
from buildbot.status.builder import SUCCESS
from buildbot.steps import shell
from buildbot.steps.transfer import FileDownload

from common import chromium_utils
import config

from master import chromium_step
from master.log_parser import gtest_command
from master.log_parser import retcode_command
from master.optional_arguments import ListProperties


# DEFAULT_TESTS is a marker to specify that the default tests should be run for
# this builder. It's mainly used for try job; this is implicit for non-try
# builders that all steps are run.
DEFAULT_TESTS = 'defaulttests'


# Performance step utils.
def CreatePerformanceStepClass(
    log_processor_class, report_link=None, output_dir=None,
    factory_properties=None, perf_name=None, test_name=None,
    command_class=None):
  """Returns ProcessLogShellStep class.

  Args:
    log_processor_class: class that will be used to process logs. Normally
      should be a subclass of process_log.PerformanceLogProcessor.
    report_link: URL that will be used as a link to results. If None,
      result won't be written into file.
    output_dir: directory where the log processor will write the results.
    command_class: command type to run for this step. Normally this will be
      chromium_step.ProcessLogShellStep.
  """
  factory_properties = factory_properties or {}
  command_class = command_class or chromium_step.ProcessLogShellStep
  # We create a log-processor class using
  # chromium_utils.InitializePartiallyWithArguments, which uses function
  # currying to create classes that have preset constructor arguments.
  # This serves two purposes:
  # 1. Allows the step to instantiate its log processor without any
  #    additional parameters;
  # 2. Creates a unique log processor class for each individual step, so
  # they can keep state that won't be shared between builds
  log_processor_class = chromium_utils.InitializePartiallyWithArguments(
      log_processor_class, report_link=report_link, output_dir=output_dir,
      factory_properties=factory_properties, perf_name=perf_name,
      test_name=test_name)
  # Similarly, we need to allow buildbot to create the step itself without
  # using additional parameters, so we create a step class that already
  # knows which log_processor to use.
  return chromium_utils.InitializePartiallyWithArguments(
      command_class, log_processor_class=log_processor_class)


class RunHooksShell(shell.ShellCommand):
  """A special run hooks shell command to allow modifying its environment
  right before it starts up."""
  def setupEnvironment(self, cmd):
    test_filters = self.getProperty('testfilter')
    test_filters = test_filters or DEFAULT_TESTS

    # If swarm tests are present ensure that the hash output required
    # by them is generated.
    if any(test.endswith('_swarm') or '_swarm:' in test
           for test in test_filters):
      environ = cmd.args.get('env', {}).copy()
      environ.setdefault('GYP_DEFINES', '')
      environ['GYP_DEFINES'] += ' test_isolation_mode=hashtable'<commit_msg>Trap exception when testfilter is not set.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Set of utilities to add commands to a buildbot factory (BuildFactory).

All the utility functions to add steps to a build factory here are not
project-specific. See the other *_commands.py for project-specific commands.
"""

import json
import ntpath
import posixpath
import re

from buildbot.locks import SlaveLock
from buildbot.process.properties import WithProperties
from buildbot.status.builder import SUCCESS
from buildbot.steps import shell
from buildbot.steps.transfer import FileDownload

from common import chromium_utils
import config

from master import chromium_step
from master.log_parser import gtest_command
from master.log_parser import retcode_command
from master.optional_arguments import ListProperties


# DEFAULT_TESTS is a marker to specify that the default tests should be run for
# this builder. It's mainly used for try job; this is implicit for non-try
# builders that all steps are run.
DEFAULT_TESTS = 'defaulttests'


# Performance step utils.
def CreatePerformanceStepClass(
    log_processor_class, report_link=None, output_dir=None,
    factory_properties=None, perf_name=None, test_name=None,
    command_class=None):
  """Returns ProcessLogShellStep class.

  Args:
    log_processor_class: class that will be used to process logs. Normally
      should be a subclass of process_log.PerformanceLogProcessor.
    report_link: URL that will be used as a link to results. If None,
      result won't be written into file.
    output_dir: directory where the log processor will write the results.
    command_class: command type to run for this step. Normally this will be
      chromium_step.ProcessLogShellStep.
  """
  factory_properties = factory_properties or {}
  command_class = command_class or chromium_step.ProcessLogShellStep
  # We create a log-processor class using
  # chromium_utils.InitializePartiallyWithArguments, which uses function
  # currying to create classes that have preset constructor arguments.
  # This serves two purposes:
  # 1. Allows the step to instantiate its log processor without any
  #    additional parameters;
  # 2. Creates a unique log processor class for each individual step, so
  # they can keep state that won't be shared between builds
  log_processor_class = chromium_utils.InitializePartiallyWithArguments(
      log_processor_class, report_link=report_link, output_dir=output_dir,
      factory_properties=factory_properties, perf_name=perf_name,
      test_name=test_name)
  # Similarly, we need to allow buildbot to create the step itself without
  # using additional parameters, so we create a step class that already
  # knows which log_processor to use.
  return chromium_utils.InitializePartiallyWithArguments(
      command_class, log_processor_class=log_processor_class)


class RunHooksShell(shell.ShellCommand):
  """A special run hooks shell command to allow modifying its environment
  right before it starts up."""
  def setupEnvironment(self, cmd):
    try:
      test_filters = self.getProperty('testfilter')
    except KeyError:
      test_filters = None
    test_filters = test_filters or DEFAULT_TESTS

    # If swarm tests are present ensure that the hash output required
    # by them is generated.
    if any(test.endswith('_swarm') or '_swarm:' in test
           for test in test_filters):
      environ = cmd.args.get('env', {}).copy()
      environ.setdefault('GYP_DEFINES', '')
      environ['GYP_DEFINES'] += ' test_isolation_mode=hashtable'
{'fraction_non_alphanumeric': 0.056434962147281484, 'fraction_numerical': 0.0016517549896765313, 'mean_word_length': 4.496217851739789, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '346095', 'n_tokens_mistral': 1952, 'n_tokens_neox': 1893, 'n_words': 778}