<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import copy

from slave import recipe_api


# Different types of builds this recipe module can do.
RECIPE_CONFIGS = {
  'chromeos_official': {
    'chromium_config': 'chromium_official',
    'chromium_apply_config': ['chromeos'],
    'gclient_config': 'chromium',
    'gclient_apply_config': ['chrome_internal'],
  },
  'chromium': {
    'chromium_config': 'chromium',
    'gclient_config': 'chromium',
  },
  'chromium_android': {
    'chromium_config': 'android',
    'gclient_config': 'chromium',
    'gclient_apply_config': ['android'],
  },
  'chromium_clang': {
    'chromium_config': 'chromium_clang',
    'gclient_config': 'chromium',
  },
  'chromium_linux_asan': {
    'chromium_config': 'chromium_linux_asan',
    'gclient_config': 'chromium',
  },
  'chromium_mac_asan': {
    'chromium_config': 'chromium_mac_asan',
    'gclient_config': 'chromium',
  },
  'chromium_chromiumos_asan': {
    'chromium_config': 'chromium_chromiumos_asan',
    'gclient_config': 'chromium',
  },
  'chromium_chromeos': {
    'chromium_config': 'chromium',
    'chromium_apply_config': ['chromeos'],
    'gclient_config': 'chromium',
  },
  'chrome_chromeos': {
    'chromium_config': 'chromium',
    'chromium_apply_config': ['chromeos', 'chrome_internal'],
    'gclient_config': 'chromium',
    'gclient_apply_config': ['chrome_internal'],
  },
  'chromium_chromeos_ozone': {
    'chromium_config': 'chromium',
    'chromium_apply_config': ['chromeos', 'ozone'],
    'gclient_config': 'chromium',
  },
  'chromium_chromeos_athena': {
    'chromium_config': 'chromium_chromeos_athena',
    'gclient_config': 'chromium',
  },
  'chromium_chromeos_clang': {
    'chromium_config': 'chromium_clang',
    'chromium_apply_config': ['chromeos'],
    'gclient_config': 'chromium',
  },
  'chromium_ios_device': {
    'chromium_config': 'chromium_ios_device',
    'gclient_config': 'ios',
  },
  'chromium_ios_ninja': {
    'chromium_config': 'chromium_ios_ninja',
    'gclient_config': 'ios',
  },
  'chromium_ios_simulator': {
    'chromium_config': 'chromium_ios_simulator',
    'gclient_config': 'ios',
  },
  'chromium_msan': {
    'chromium_config': 'chromium_msan',
    'gclient_config': 'chromium',
  },
  'chromium_no_goma': {
    'chromium_config': 'chromium_no_goma',
    'gclient_config': 'chromium',
  },
  'chromium_oilpan': {
    'chromium_config': 'chromium_official',
    'chromium_apply_config': ['oilpan'],
    'gclient_config': 'chromium',
    'gclient_apply_config': ['chrome_internal'],
  },
  'chromium_v8': {
    'chromium_config': 'chromium',
    'gclient_config': 'chromium',
    'gclient_apply_config': [
      'v8_bleeding_edge_git',
      'chromium_lkcr',
      'show_v8_revision',
    ],
  },
  'chromium_skia': {
    'chromium_config': 'chromium',
    'gclient_config': 'chromium_skia',
  },
  'official': {
    'chromium_config': 'chromium_official',
    'gclient_config': 'chromium',
    'gclient_apply_config': ['chrome_internal'],
  },
  'perf': {
    'chromium_config': 'chromium_official',
    'gclient_config': 'perf',
  }
}


class ChromiumTestsApi(recipe_api.RecipeApi):
  def sync_and_configure_build(self, mastername, buildername,
                               override_bot_type=None, enable_swarming=False,
                               chromium_apply_config=None):
    # Make an independent copy so that we don't overwrite global state
    # with updates made dynamically based on the test specs.
    master_dict = copy.deepcopy(self.m.chromium.builders.get(mastername, {}))

    bot_config = master_dict.get('builders', {}).get(buildername)
    master_config = master_dict.get('settings', {})
    recipe_config_name = bot_config['recipe_config']
    assert recipe_config_name, (
        'Unrecognized builder name %r for master %r.' % (
            buildername, mastername))
    recipe_config = RECIPE_CONFIGS[recipe_config_name]

    self.m.chromium.set_config(
        recipe_config['chromium_config'],
        **bot_config.get('chromium_config_kwargs', {}))
    # Set GYP_DEFINES explicitly because chromium config constructor does
    # not support that.
    self.m.chromium.c.gyp_env.GYP_DEFINES.update(
        bot_config.get('GYP_DEFINES', {}))
    if bot_config.get('use_isolate'):
      self.m.isolate.set_isolate_environment(self.m.chromium.c)
    for c in recipe_config.get('chromium_apply_config', []):
      self.m.chromium.apply_config(c)
    for c in bot_config.get('chromium_apply_config', []):
      self.m.chromium.apply_config(c)
    if chromium_apply_config:
      for c in chromium_apply_config:
        self.m.chromium.apply_config(c)
    self.m.gclient.set_config(
        recipe_config['gclient_config'],
        **bot_config.get('gclient_config_kwargs', {}))
    for c in recipe_config.get('gclient_apply_config', []):
      self.m.gclient.apply_config(c)
    for c in bot_config.get('gclient_apply_config', []):
      self.m.gclient.apply_config(c)

    if 'android_config' in bot_config:
      self.m.chromium_android.set_config(
          bot_config['android_config'],
          **bot_config.get('chromium_config_kwargs', {}))

    bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester')

    if bot_config.get('set_component_rev'):
      # If this is a component build and the main revision is e.g. blink,
      # webrtc, or v8, the custom deps revision of this component must be
      # dynamically set to either:
      # (1) the revision of the builder if this is a tester,
      # (2) 'revision' from the waterfall, or
      # (3) 'HEAD' for forced builds with unspecified 'revision'.
      # TODO(machenbach): Use parent_got_cr_revision on testers with component
      # builds to match also the chromium revision from the builder.
      component_rev = self.m.properties.get('revision', 'HEAD')
      if bot_type == 'tester':
        component_rev = self.m.properties.get(
            'parent_got_revision', component_rev)
      dep = bot_config.get('set_component_rev')
      self.m.gclient.c.revisions[dep['name']] = dep['rev_str'] % component_rev

    if self.m.platform.is_win:
      self.m.chromium.taskkill()

    # Bot Update re-uses the gclient configs.
    update_step = self.m.bot_update.ensure_checkout()
    assert update_step.json.output['did_run']
    # HACK(dnj): Remove after 'crbug.com/398105' has landed
    self.m.chromium.set_build_properties(update_step.json.output['properties'])

    if not enable_swarming:
      enable_swarming = bot_config.get('enable_swarming')

    if enable_swarming:
      self.m.isolate.set_isolate_environment(self.m.chromium.c)
      self.m.swarming.check_client_version()
      self.m.swarming.default_priority = 50
      os_dimension = bot_config.get('swarming_dimensions', {}).get('os')
      if os_dimension:
        self.m.swarming.set_default_dimension('os', os_dimension)
      cpu_dimension = bot_config.get('swarming_dimensions', {}).get('cpu')
      if cpu_dimension:
        self.m.swarming.set_default_dimension('cpu', cpu_dimension)

    if not bot_config.get('disable_runhooks'):
      if self.m.tryserver.is_tryserver:
        try:
          self.m.chromium.runhooks(name='runhooks (with patch)')
        except self.m.step.StepFailure:
          # As part of deapplying patch we call runhooks without the patch.
          self.deapply_patch(update_step)
          raise
      else:
        self.m.chromium.runhooks()

    test_spec_file = bot_config.get('testing', {}).get('test_spec_file',
                                                       '%s.json' % mastername)
    test_spec_path = self.m.path['checkout'].join('testing', 'buildbot',
                                               test_spec_file)
    # TODO(phajdan.jr): Bots should have no generators instead.<commit_msg>Remove redundant swarming default_priority assignment<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import copy

from slave import recipe_api


# Different types of builds this recipe module can do.
RECIPE_CONFIGS = {
  'chromeos_official': {
    'chromium_config': 'chromium_official',
    'chromium_apply_config': ['chromeos'],
    'gclient_config': 'chromium',
    'gclient_apply_config': ['chrome_internal'],
  },
  'chromium': {
    'chromium_config': 'chromium',
    'gclient_config': 'chromium',
  },
  'chromium_android': {
    'chromium_config': 'android',
    'gclient_config': 'chromium',
    'gclient_apply_config': ['android'],
  },
  'chromium_clang': {
    'chromium_config': 'chromium_clang',
    'gclient_config': 'chromium',
  },
  'chromium_linux_asan': {
    'chromium_config': 'chromium_linux_asan',
    'gclient_config': 'chromium',
  },
  'chromium_mac_asan': {
    'chromium_config': 'chromium_mac_asan',
    'gclient_config': 'chromium',
  },
  'chromium_chromiumos_asan': {
    'chromium_config': 'chromium_chromiumos_asan',
    'gclient_config': 'chromium',
  },
  'chromium_chromeos': {
    'chromium_config': 'chromium',
    'chromium_apply_config': ['chromeos'],
    'gclient_config': 'chromium',
  },
  'chrome_chromeos': {
    'chromium_config': 'chromium',
    'chromium_apply_config': ['chromeos', 'chrome_internal'],
    'gclient_config': 'chromium',
    'gclient_apply_config': ['chrome_internal'],
  },
  'chromium_chromeos_ozone': {
    'chromium_config': 'chromium',
    'chromium_apply_config': ['chromeos', 'ozone'],
    'gclient_config': 'chromium',
  },
  'chromium_chromeos_athena': {
    'chromium_config': 'chromium_chromeos_athena',
    'gclient_config': 'chromium',
  },
  'chromium_chromeos_clang': {
    'chromium_config': 'chromium_clang',
    'chromium_apply_config': ['chromeos'],
    'gclient_config': 'chromium',
  },
  'chromium_ios_device': {
    'chromium_config': 'chromium_ios_device',
    'gclient_config': 'ios',
  },
  'chromium_ios_ninja': {
    'chromium_config': 'chromium_ios_ninja',
    'gclient_config': 'ios',
  },
  'chromium_ios_simulator': {
    'chromium_config': 'chromium_ios_simulator',
    'gclient_config': 'ios',
  },
  'chromium_msan': {
    'chromium_config': 'chromium_msan',
    'gclient_config': 'chromium',
  },
  'chromium_no_goma': {
    'chromium_config': 'chromium_no_goma',
    'gclient_config': 'chromium',
  },
  'chromium_oilpan': {
    'chromium_config': 'chromium_official',
    'chromium_apply_config': ['oilpan'],
    'gclient_config': 'chromium',
    'gclient_apply_config': ['chrome_internal'],
  },
  'chromium_v8': {
    'chromium_config': 'chromium',
    'gclient_config': 'chromium',
    'gclient_apply_config': [
      'v8_bleeding_edge_git',
      'chromium_lkcr',
      'show_v8_revision',
    ],
  },
  'chromium_skia': {
    'chromium_config': 'chromium',
    'gclient_config': 'chromium_skia',
  },
  'official': {
    'chromium_config': 'chromium_official',
    'gclient_config': 'chromium',
    'gclient_apply_config': ['chrome_internal'],
  },
  'perf': {
    'chromium_config': 'chromium_official',
    'gclient_config': 'perf',
  }
}


class ChromiumTestsApi(recipe_api.RecipeApi):
  def sync_and_configure_build(self, mastername, buildername,
                               override_bot_type=None, enable_swarming=False,
                               chromium_apply_config=None):
    # Make an independent copy so that we don't overwrite global state
    # with updates made dynamically based on the test specs.
    master_dict = copy.deepcopy(self.m.chromium.builders.get(mastername, {}))

    bot_config = master_dict.get('builders', {}).get(buildername)
    master_config = master_dict.get('settings', {})
    recipe_config_name = bot_config['recipe_config']
    assert recipe_config_name, (
        'Unrecognized builder name %r for master %r.' % (
            buildername, mastername))
    recipe_config = RECIPE_CONFIGS[recipe_config_name]

    self.m.chromium.set_config(
        recipe_config['chromium_config'],
        **bot_config.get('chromium_config_kwargs', {}))
    # Set GYP_DEFINES explicitly because chromium config constructor does
    # not support that.
    self.m.chromium.c.gyp_env.GYP_DEFINES.update(
        bot_config.get('GYP_DEFINES', {}))
    if bot_config.get('use_isolate'):
      self.m.isolate.set_isolate_environment(self.m.chromium.c)
    for c in recipe_config.get('chromium_apply_config', []):
      self.m.chromium.apply_config(c)
    for c in bot_config.get('chromium_apply_config', []):
      self.m.chromium.apply_config(c)
    if chromium_apply_config:
      for c in chromium_apply_config:
        self.m.chromium.apply_config(c)
    self.m.gclient.set_config(
        recipe_config['gclient_config'],
        **bot_config.get('gclient_config_kwargs', {}))
    for c in recipe_config.get('gclient_apply_config', []):
      self.m.gclient.apply_config(c)
    for c in bot_config.get('gclient_apply_config', []):
      self.m.gclient.apply_config(c)

    if 'android_config' in bot_config:
      self.m.chromium_android.set_config(
          bot_config['android_config'],
          **bot_config.get('chromium_config_kwargs', {}))

    bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester')

    if bot_config.get('set_component_rev'):
      # If this is a component build and the main revision is e.g. blink,
      # webrtc, or v8, the custom deps revision of this component must be
      # dynamically set to either:
      # (1) the revision of the builder if this is a tester,
      # (2) 'revision' from the waterfall, or
      # (3) 'HEAD' for forced builds with unspecified 'revision'.
      # TODO(machenbach): Use parent_got_cr_revision on testers with component
      # builds to match also the chromium revision from the builder.
      component_rev = self.m.properties.get('revision', 'HEAD')
      if bot_type == 'tester':
        component_rev = self.m.properties.get(
            'parent_got_revision', component_rev)
      dep = bot_config.get('set_component_rev')
      self.m.gclient.c.revisions[dep['name']] = dep['rev_str'] % component_rev

    if self.m.platform.is_win:
      self.m.chromium.taskkill()

    # Bot Update re-uses the gclient configs.
    update_step = self.m.bot_update.ensure_checkout()
    assert update_step.json.output['did_run']
    # HACK(dnj): Remove after 'crbug.com/398105' has landed
    self.m.chromium.set_build_properties(update_step.json.output['properties'])

    if not enable_swarming:
      enable_swarming = bot_config.get('enable_swarming')

    if enable_swarming:
      self.m.isolate.set_isolate_environment(self.m.chromium.c)
      self.m.swarming.check_client_version()
      os_dimension = bot_config.get('swarming_dimensions', {}).get('os')
      if os_dimension:
        self.m.swarming.set_default_dimension('os', os_dimension)
      cpu_dimension = bot_config.get('swarming_dimensions', {}).get('cpu')
      if cpu_dimension:
        self.m.swarming.set_default_dimension('cpu', cpu_dimension)

    if not bot_config.get('disable_runhooks'):
      if self.m.tryserver.is_tryserver:
        try:
          self.m.chromium.runhooks(name='runhooks (with patch)')
        except self.m.step.StepFailure:
          # As part of deapplying patch we call runhooks without the patch.
          self.deapply_patch(update_step)
          raise
      else:
        self.m.chromium.runhooks()

    test_spec_file = bot_config.get('testing', {}).get('test_spec_file',
                                                       '%s.json' % mastername)
    test_spec_path = self.m.path['checkout'].join('testing', 'buildbot',
                                               test_spec_file)
    # TODO(phajdan.jr): Bots should have no generators instead.
{'fraction_non_alphanumeric': 0.13095689546201492, 'fraction_numerical': 0.0022753128555176336, 'mean_word_length': 3.569159688131678, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7243440', 'n_tokens_mistral': 5607, 'n_tokens_neox': 5258, 'n_words': 939}