<commit_before>#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# TODO(hinoka): Use logging.

import cStringIO
import codecs
import copy
import ctypes
import json
import optparse
import os
import pprint
import re
import socket
import subprocess
import sys
import tempfile
import threading
import time
import urllib2
import urlparse
import uuid

import os.path as path

# How many bytes at a time to read from pipes.
BUF_SIZE = 256

# Set this to true on flag day.
FLAG_DAY = False

# Should we upload the status of this step?
UPLOAD_TELEMETRY = True

# Define a bunch of directory paths.
# Relative to the current working directory.
CURRENT_DIR = path.abspath(os.getcwd())
BUILDER_DIR = path.dirname(CURRENT_DIR)
SLAVE_DIR = path.dirname(BUILDER_DIR)

# Relative to this script's filesystem path.
THIS_DIR = path.dirname(path.abspath(__file__))
SCRIPTS_DIR = path.dirname(THIS_DIR)
BUILD_DIR = path.dirname(SCRIPTS_DIR)
ROOT_DIR = path.dirname(BUILD_DIR)
BUILD_INTERNAL_DIR = path.join(ROOT_DIR, 'build_internal')
DEPOT_TOOLS_DIR = path.join(ROOT_DIR, 'depot_tools')


CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'

# Official builds use buildspecs, so this is a special case.
BUILDSPEC_RE = r'^/chrome-internal/trunk/tools/buildspec/build/(.*)$'
GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/'
                      'buildspec')

# This is the repository that the buildspecs2git cron job mirrors
# all buildspecs into. When we see an svn buildspec, we rely on the
# buildspecs2git cron job to produce the git version of the buildspec.
GIT_BUILDSPEC_REPO = (
    'https://chrome-internal.googlesource.com/chrome/tools/git_buildspecs')

# Copied from scripts/recipes/chromium.py.
GOT_REVISION_MAPPINGS = {
    '/chrome/trunk/src': {
        'src/': 'got_revision',
        'src/native_client/': 'got_nacl_revision',
        'src/tools/swarm_client/': 'got_swarm_client_revision',
        'src/tools/swarming_client/': 'got_swarming_client_revision',
        'src/third_party/WebKit/': 'got_webkit_revision',
        'src/third_party/webrtc/': 'got_webrtc_revision',
        'src/v8/': 'got_v8_revision',
    }
}


BOT_UPDATE_MESSAGE = """
What is the "Bot Update" step?
==============================

This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
its dependencies) is checked out in a consistent state. This means that all of
the necessary repositories are checked out, no extra repositories are checked
out, and no locally modified files are present.

These actions used to be taken care of by the "gclient revert" and "update"
steps. However, those steps are known to be buggy and occasionally flaky. This
step has two main advantages over them:
  * it only operates in Git, so the logic can be clearer and cleaner; and
  * it is a slave-side script, so its behavior can be modified without
    restarting the master.

Why Git, you ask? Because that is the direction that the Chromium project is
heading. This step is an integral part of the transition from using the SVN repo
at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while
we fully convert everything to Git. This message will get out of your way
eventually, and the waterfall will be a happier place because of it.

This step can be activated or deactivated independently on every builder on
every master. When it is active, the "gclient revert" and "update" steps become
no-ops. When it is inactive, it prints this message, cleans up after itself, and
lets everything else continue as though nothing has changed. Eventually, when
everything is stable enough, this step will replace them entirely.

Debugging information:
(master/builder/slave may be unspecified on recipes)
master: %(master)s
builder: %(builder)s
slave: %(slave)s
forced by recipes: %(recipe)s
bot_update.py is:"""

ACTIVATED_MESSAGE = """ACTIVE.
The bot will perform a Git checkout in this step.
The "gclient revert" and "update" steps are no-ops.

"""

NOT_ACTIVATED_MESSAGE = """INACTIVE.
This step does nothing. You actually want to look at the "update" step.

"""


GCLIENT_TEMPLATE = """solutions = %(solutions)s

cache_dir = %(cache_dir)s
%(target_os)s
%(target_os_only)s
"""


internal_data = {}
if os.path.isdir(BUILD_INTERNAL_DIR):
  local_vars = {}
  try:
    execfile(os.path.join(
        BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'),
        local_vars)
  except Exception:
    # Same as if BUILD_INTERNAL_DIR didn't exist in the first place.
    print 'Warning: unable to read internal configuration file.'
    print 'If this is an internal bot, this step may be erroneously inactive.'
  internal_data = local_vars

RECOGNIZED_PATHS = {
    # If SVN path matches key, the entire URL is rewritten to the Git url.
    '/chrome/trunk/src':
        CHROMIUM_SRC_URL,
    '/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS':
        CHROMIUM_GIT_HOST + '/chromium/deps/webrtc/webrtc.DEPS.git',
    '/svn/branches/bleeding_edge': CHROMIUM_GIT_HOST + '/external/v8.git',
}
RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {}))

ENABLED_MASTERS = [
    'bot_update.always_on',
    'chromium.chrome',
    'chromium.chromedriver',
    'chromium.chromiumos',
    'chromium.endure',
    'chromium',
    'chromium.fyi',
    'chromium.git',
    'chromium.gpu',
    'chromium.gpu.fyi',
    'chromium.linux',
    'chromium.mac',
    'chromium.memory',
    'chromium.memory.fyi',
    'chromium.perf',
    'chromium.swarm',
    'chromium.webkit',
    'chromium.webrtc',
    'chromium.webrtc.fyi',
    'chromium.win',
    'client.drmemory',
    'client.nacl.sdk',
    'client.nacl.sdk.mono',
    'tryserver.blink',
    'tryserver.chromium',
    'chromium.perf.fyi',
]
ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', [])

ENABLED_BUILDERS = {
    'client.v8': [
        'Linux Debug Builder',
        'Linux Tests (dbg)(1)',
        'V8 Linux - x87 - nosnap - debug',
        'V8 Linux - vtunejit',
        'V8 Linux - predictable',
        'NaCl V8 Linux',
        'V8 Linux - noi18n - debug',
        'V8 Linux - debug builder',
        'V8 Linux - full debug builder',
        'V8 Linux - debug',
        'V8 Linux - debug - isolates',
        'V8 Linux - debug - nosse3',
        'V8 Linux - debug - nosse4',
        'V8 GC Stress - 1',
        'V8 GC Stress - 2',
        'V8 GC Stress - 3',
        'V8 Win32 - builder',
        'V8 Win32 - 1',
        'V8 Win32 - 2',
        'V8 Win32 - debug builder',
        'V8 Win32 - full debug builder',
        'V8 Win32 - debug - 1',
        'V8 Win32 - debug - 2',
        'V8 Win32 - debug - 3',
        'V8 Win32 - nosnap - shared',
        'V8 Win64',
        'V8 Mac',
        'V8 Mac - debug',
        'V8 Mac64',
        'V8 Mac64 - debug',
        'V8 Arm - builder',
        'V8 Arm - debug builder',
        'V8 Arm',
        'V8 Arm - debug',
        'V8 Arm GC Stress',
    ],
    'tryserver.chromium.gpu': [
        'linux_gpu',
        'mac_gpu',
    ],
    'chromium.lkgr': [
        'Android',
        'Linux x64',
        'Linux',
        'Mac',
        'Telemetry Harness Upload',
        'ASAN Release',
        'ASAN Release (symbolized)',
        'ASAN Debug',
        'ASan Debug (32-bit x86 with V8-ARM)',
        'ASan Release (32-bit x86 with V8-ARM)',
        'ASan Release (32-bit x86 with V8-ARM, symbolized)',
        'TSAN Release',
        'TSAN Debug',
        'Win ASan Release',
        'Mac ASAN Debug',
        'Mac ASAN Release',
    ],
    'client.v8.branches': [
        'Chromium ASAN (symbolized) - trunk',
        'Chromium ASAN - trunk - debug',
        'Chromium Win SyzyASAN - trunk',
    ],
}
ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {}))

ENABLED_SLAVES = {
    # This is enabled on a bot-to-bot basis to ensure that we don't have
    # bots that have mixed configs.
    'chromium.fyi': [
        'build1-m1',  # Chromium Builder / Chromium Builder (dbg)
        'vm928-m1',   # Chromium Linux Buildrunner
        'vm859-m1',   # Chromium Linux Redux
        'vm933-m1',   # ChromiumOS Linux Tests
    ],

    # Tryserver bots need to be enabled on a bot basis to make sure checkouts
    # on the same bot do not conflict.
    'tryserver.chromium': ['slave%d-c4' % i for i in range(250, 400)] +
                          ['slave%d-c4' % i for i in range(102, 121)] +
                          ['vm%d-m4' % i for i in [468, 469, 497, 502, 503]] +
                          ['vm%d-m4' % i for i in range(800, 810)] +
                          ['vm%d-m4' % i for i in range(666, 671)] +
                          ['build%d-a4' % i for i in range(100, 140)],
}
ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {}))

# Disabled filters get run AFTER enabled filters, so for example if a builder
# config is enabled, but a bot on that builder is disabled, that bot will
# be disabled.
DISABLED_BUILDERS = {
    'chromium.gpu': [<commit_msg>Partial revert of Enable 'bot_update' on ASAN builders.<commit_after>#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# TODO(hinoka): Use logging.

import cStringIO
import codecs
import copy
import ctypes
import json
import optparse
import os
import pprint
import re
import socket
import subprocess
import sys
import tempfile
import threading
import time
import urllib2
import urlparse
import uuid

import os.path as path

# How many bytes at a time to read from pipes.
BUF_SIZE = 256

# Set this to true on flag day.
FLAG_DAY = False

# Should we upload the status of this step?
UPLOAD_TELEMETRY = True

# Define a bunch of directory paths.
# Relative to the current working directory.
CURRENT_DIR = path.abspath(os.getcwd())
BUILDER_DIR = path.dirname(CURRENT_DIR)
SLAVE_DIR = path.dirname(BUILDER_DIR)

# Relative to this script's filesystem path.
THIS_DIR = path.dirname(path.abspath(__file__))
SCRIPTS_DIR = path.dirname(THIS_DIR)
BUILD_DIR = path.dirname(SCRIPTS_DIR)
ROOT_DIR = path.dirname(BUILD_DIR)
BUILD_INTERNAL_DIR = path.join(ROOT_DIR, 'build_internal')
DEPOT_TOOLS_DIR = path.join(ROOT_DIR, 'depot_tools')


CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'

# Official builds use buildspecs, so this is a special case.
BUILDSPEC_RE = r'^/chrome-internal/trunk/tools/buildspec/build/(.*)$'
GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/'
                      'buildspec')

# This is the repository that the buildspecs2git cron job mirrors
# all buildspecs into. When we see an svn buildspec, we rely on the
# buildspecs2git cron job to produce the git version of the buildspec.
GIT_BUILDSPEC_REPO = (
    'https://chrome-internal.googlesource.com/chrome/tools/git_buildspecs')

# Copied from scripts/recipes/chromium.py.
GOT_REVISION_MAPPINGS = {
    '/chrome/trunk/src': {
        'src/': 'got_revision',
        'src/native_client/': 'got_nacl_revision',
        'src/tools/swarm_client/': 'got_swarm_client_revision',
        'src/tools/swarming_client/': 'got_swarming_client_revision',
        'src/third_party/WebKit/': 'got_webkit_revision',
        'src/third_party/webrtc/': 'got_webrtc_revision',
        'src/v8/': 'got_v8_revision',
    }
}


BOT_UPDATE_MESSAGE = """
What is the "Bot Update" step?
==============================

This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
its dependencies) is checked out in a consistent state. This means that all of
the necessary repositories are checked out, no extra repositories are checked
out, and no locally modified files are present.

These actions used to be taken care of by the "gclient revert" and "update"
steps. However, those steps are known to be buggy and occasionally flaky. This
step has two main advantages over them:
  * it only operates in Git, so the logic can be clearer and cleaner; and
  * it is a slave-side script, so its behavior can be modified without
    restarting the master.

Why Git, you ask? Because that is the direction that the Chromium project is
heading. This step is an integral part of the transition from using the SVN repo
at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while
we fully convert everything to Git. This message will get out of your way
eventually, and the waterfall will be a happier place because of it.

This step can be activated or deactivated independently on every builder on
every master. When it is active, the "gclient revert" and "update" steps become
no-ops. When it is inactive, it prints this message, cleans up after itself, and
lets everything else continue as though nothing has changed. Eventually, when
everything is stable enough, this step will replace them entirely.

Debugging information:
(master/builder/slave may be unspecified on recipes)
master: %(master)s
builder: %(builder)s
slave: %(slave)s
forced by recipes: %(recipe)s
bot_update.py is:"""

ACTIVATED_MESSAGE = """ACTIVE.
The bot will perform a Git checkout in this step.
The "gclient revert" and "update" steps are no-ops.

"""

NOT_ACTIVATED_MESSAGE = """INACTIVE.
This step does nothing. You actually want to look at the "update" step.

"""


GCLIENT_TEMPLATE = """solutions = %(solutions)s

cache_dir = %(cache_dir)s
%(target_os)s
%(target_os_only)s
"""


internal_data = {}
if os.path.isdir(BUILD_INTERNAL_DIR):
  local_vars = {}
  try:
    execfile(os.path.join(
        BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'),
        local_vars)
  except Exception:
    # Same as if BUILD_INTERNAL_DIR didn't exist in the first place.
    print 'Warning: unable to read internal configuration file.'
    print 'If this is an internal bot, this step may be erroneously inactive.'
  internal_data = local_vars

RECOGNIZED_PATHS = {
    # If SVN path matches key, the entire URL is rewritten to the Git url.
    '/chrome/trunk/src':
        CHROMIUM_SRC_URL,
    '/chrome/trunk/deps/third_party/webrtc/webrtc.DEPS':
        CHROMIUM_GIT_HOST + '/chromium/deps/webrtc/webrtc.DEPS.git',
    '/svn/branches/bleeding_edge': CHROMIUM_GIT_HOST + '/external/v8.git',
}
RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {}))

ENABLED_MASTERS = [
    'bot_update.always_on',
    'chromium.chrome',
    'chromium.chromedriver',
    'chromium.chromiumos',
    'chromium.endure',
    'chromium',
    'chromium.fyi',
    'chromium.git',
    'chromium.gpu',
    'chromium.gpu.fyi',
    'chromium.linux',
    'chromium.mac',
    'chromium.memory',
    'chromium.memory.fyi',
    'chromium.perf',
    'chromium.swarm',
    'chromium.webkit',
    'chromium.webrtc',
    'chromium.webrtc.fyi',
    'chromium.win',
    'client.drmemory',
    'client.nacl.sdk',
    'client.nacl.sdk.mono',
    'tryserver.blink',
    'tryserver.chromium',
    'chromium.perf.fyi',
]
ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', [])

ENABLED_BUILDERS = {
    'client.v8': [
        'Linux Debug Builder',
        'Linux Tests (dbg)(1)',
        'V8 Linux - x87 - nosnap - debug',
        'V8 Linux - vtunejit',
        'V8 Linux - predictable',
        'NaCl V8 Linux',
        'V8 Linux - noi18n - debug',
        'V8 Linux - debug builder',
        'V8 Linux - full debug builder',
        'V8 Linux - debug',
        'V8 Linux - debug - isolates',
        'V8 Linux - debug - nosse3',
        'V8 Linux - debug - nosse4',
        'V8 GC Stress - 1',
        'V8 GC Stress - 2',
        'V8 GC Stress - 3',
        'V8 Win32 - builder',
        'V8 Win32 - 1',
        'V8 Win32 - 2',
        'V8 Win32 - debug builder',
        'V8 Win32 - full debug builder',
        'V8 Win32 - debug - 1',
        'V8 Win32 - debug - 2',
        'V8 Win32 - debug - 3',
        'V8 Win32 - nosnap - shared',
        'V8 Win64',
        'V8 Mac',
        'V8 Mac - debug',
        'V8 Mac64',
        'V8 Mac64 - debug',
        'V8 Arm - builder',
        'V8 Arm - debug builder',
        'V8 Arm',
        'V8 Arm - debug',
        'V8 Arm GC Stress',
    ],
    'tryserver.chromium.gpu': [
        'linux_gpu',
        'mac_gpu',
    ],
    'chromium.lkgr': [
        'Android',
        'Linux x64',
        'Linux',
        'Mac',
        'Telemetry Harness Upload',
        'ASAN Release',
        'ASAN Release (symbolized)',
        'ASAN Debug',
        'ASan Debug (32-bit x86 with V8-ARM)',
        'ASan Release (32-bit x86 with V8-ARM)',
        'ASan Release (32-bit x86 with V8-ARM, symbolized)',
        'TSAN Release',
        'TSAN Debug',
        'Win ASan Release',
        'Mac ASAN Debug',
        'Mac ASAN Release',
    ],
}
ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {}))

ENABLED_SLAVES = {
    # This is enabled on a bot-to-bot basis to ensure that we don't have
    # bots that have mixed configs.
    'chromium.fyi': [
        'build1-m1',  # Chromium Builder / Chromium Builder (dbg)
        'vm928-m1',   # Chromium Linux Buildrunner
        'vm859-m1',   # Chromium Linux Redux
        'vm933-m1',   # ChromiumOS Linux Tests
    ],

    # Tryserver bots need to be enabled on a bot basis to make sure checkouts
    # on the same bot do not conflict.
    'tryserver.chromium': ['slave%d-c4' % i for i in range(250, 400)] +
                          ['slave%d-c4' % i for i in range(102, 121)] +
                          ['vm%d-m4' % i for i in [468, 469, 497, 502, 503]] +
                          ['vm%d-m4' % i for i in range(800, 810)] +
                          ['vm%d-m4' % i for i in range(666, 671)] +
                          ['build%d-a4' % i for i in range(100, 140)],
}
ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {}))

# Disabled filters get run AFTER enabled filters, so for example if a builder
# config is enabled, but a bot on that builder is disabled, that bot will
# be disabled.
DISABLED_BUILDERS = {
    'chromium.gpu': [
{'fraction_non_alphanumeric': 0.11228263869721225, 'fraction_numerical': 0.018603367375103506, 'mean_word_length': 2.897590361445783, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1524890', 'n_tokens_mistral': 6298, 'n_tokens_neox': 5976, 'n_words': 1890}