<commit_before>#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Enable 'with' statements in Python 2.5
from __future__ import with_statement

import os.path
import re
import subprocess
import sys
import time

from buildbot_lib import (
    BuildContext, BuildStatus, Command, EnsureDirectoryExists,
    ParseStandardCommandLine, RemoveDirectory, RunBuild, SCons, Step, StepLink,
    StepText, TryToCleanContents)


# Windows-specific environment manipulation
def SetupWindowsEnvironment(context):
  # Blow away path for now if on the bots (to be more hermetic).
  if os.environ.get('BUILDBOT_SLAVENAME'):
    paths = [
        r'c:\b\depot_tools',
        r'c:\b\depot_tools\python_bin',
        r'c:\b\build_internal\tools',
        r'e:\b\depot_tools',
        r'e:\b\depot_tools\python_bin',
        r'e:\b\build_internal\tools',
        r'C:\WINDOWS\system32',
        r'C:\WINDOWS\system32\WBEM',
        ]
    context.SetEnv('PATH', os.pathsep.join(paths))

  # Poke around looking for MSVC.  We should do something more principled in
  # the future.

  # The name of Program Files can differ, depending on the bittage of Windows.
  program_files = r'c:\Program Files (x86)'
  if not os.path.exists(program_files):
    program_files = r'c:\Program Files'
    if not os.path.exists(program_files):
      raise Exception('Cannot find the Program Files directory!')

  # The location of MSVC can differ depending on the version.
  msvc_locs = [
      ('Microsoft Visual Studio 10.0', 'VS100COMNTOOLS', '2010'),
      ('Microsoft Visual Studio 9.0', 'VS90COMNTOOLS', '2008'),
      ('Microsoft Visual Studio 8.0', 'VS80COMNTOOLS', '2005'),
  ]

  for dirname, comntools_var, gyp_msvs_version in msvc_locs:
    msvc = os.path.join(program_files, dirname)
    context.SetEnv('GYP_MSVS_VERSION', gyp_msvs_version)
    if os.path.exists(msvc):
      break
  else:
    # The break statement did not execute.
    raise Exception('Cannot find MSVC!')

  # Put MSVC in the path.
  vc = os.path.join(msvc, 'VC')
  comntools = os.path.join(msvc, 'Common7', 'Tools')
  perf = os.path.join(msvc, 'Team Tools', 'Performance Tools')
  context.SetEnv('PATH', os.pathsep.join([
      context.GetEnv('PATH'),
      vc,
      comntools,
      perf]))

  # SCons needs this variable to find vsvars.bat.
  # The end slash is needed because the batch files expect it.
  context.SetEnv(comntools_var, comntools + '\\')

  # This environment variable will SCons to print debug info while it searches
  # for MSVC.
  context.SetEnv('SCONS_MSCOMMON_DEBUG', '-')

  # Needed for finding devenv.
  context['msvc'] = msvc

  # The context on other systems has GYP_DEFINES set, set it for windows to be
  # able to save and restore without KeyError.
  context.SetEnv('GYP_DEFINES', '')


def SetupGypDefines(context, extra_vars=[]):
  context.SetEnv('GYP_DEFINES', ' '.join(context['gyp_vars'] + extra_vars))


def SetupLinuxEnvironment(context):
  SetupGypDefines(context, ['target_arch='+context['gyp_arch']])


def SetupMacEnvironment(context):
  SetupGypDefines(context)
  context.SetEnv('GYP_GENERATORS', 'ninja')


def SetupContextVars(context):
  # The branch is set to native_client on the main bots, on the trybots it's
  # set to ''.  Otherwise, we should assume a particular branch is being used.
  context['branch'] = os.environ.get('BUILDBOT_BRANCH', 'native_client')
  context['off_trunk'] = context['branch'] not in ['native_client', '']


def ValidatorTest(context, architecture, validator, warn_only=False):
  cmd = [
      sys.executable,
      'tests/abi_corpus/validator_regression_test.py',
      '--keep-going',
      '--validator', validator,
      '--arch', architecture
  ]
  if warn_only:
    cmd.append('--warn-only')
  Command(context, cmd=cmd)


def SummarizeCoverage(context):
  Command(context, [
      sys.executable,
      'tools/coverage_summary.py',
      context['platform'] + '-' + context['default_scons_platform'],
  ])


def ArchiveCoverage(context):
  gsutil = '../../../../third_party/gsutil/gsutil'
  gsd_url = 'http://gsdview.appspot.com/nativeclient-coverage2/revs'
  variant_name = ('coverage-' + context['platform'] + '-' +
                  context['default_scons_platform'])
  coverage_path = variant_name + '/html/index.html'
  revision = os.environ.get('BUILDBOT_REVISION', 'None')
  link_url = gsd_url + '/' + revision + '/' + coverage_path
  gsd_base = 'gs://nativeclient-coverage2/revs'
  gs_path = gsd_base + '/' + revision + '/' + variant_name
  cov_dir = 'scons-out/' + variant_name + '/coverage'
  # Copy lcov file.
  Command(context, [
      sys.executable, gsutil,
      'cp', '-a', 'public-read',
      cov_dir + '/coverage.lcov',
      gs_path + '/coverage.lcov',
  ])
  # Copy html.
  Command(context, [
      sys.executable, gsutil,
      'cp', '-R', '-a', 'public-read',
      'html', gs_path,
  ], cwd=cov_dir)
  print '@@@STEP_LINK@view@%s@@@' % link_url
<commit_msg>Use absolute path to gsutil in coverage archive.<commit_after>#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Enable 'with' statements in Python 2.5
from __future__ import with_statement

import os.path
import re
import subprocess
import sys
import time

from buildbot_lib import (
    BuildContext, BuildStatus, Command, EnsureDirectoryExists,
    ParseStandardCommandLine, RemoveDirectory, RunBuild, SCons, Step, StepLink,
    StepText, TryToCleanContents)


# Windows-specific environment manipulation
def SetupWindowsEnvironment(context):
  # Blow away path for now if on the bots (to be more hermetic).
  if os.environ.get('BUILDBOT_SLAVENAME'):
    paths = [
        r'c:\b\depot_tools',
        r'c:\b\depot_tools\python_bin',
        r'c:\b\build_internal\tools',
        r'e:\b\depot_tools',
        r'e:\b\depot_tools\python_bin',
        r'e:\b\build_internal\tools',
        r'C:\WINDOWS\system32',
        r'C:\WINDOWS\system32\WBEM',
        ]
    context.SetEnv('PATH', os.pathsep.join(paths))

  # Poke around looking for MSVC.  We should do something more principled in
  # the future.

  # The name of Program Files can differ, depending on the bittage of Windows.
  program_files = r'c:\Program Files (x86)'
  if not os.path.exists(program_files):
    program_files = r'c:\Program Files'
    if not os.path.exists(program_files):
      raise Exception('Cannot find the Program Files directory!')

  # The location of MSVC can differ depending on the version.
  msvc_locs = [
      ('Microsoft Visual Studio 10.0', 'VS100COMNTOOLS', '2010'),
      ('Microsoft Visual Studio 9.0', 'VS90COMNTOOLS', '2008'),
      ('Microsoft Visual Studio 8.0', 'VS80COMNTOOLS', '2005'),
  ]

  for dirname, comntools_var, gyp_msvs_version in msvc_locs:
    msvc = os.path.join(program_files, dirname)
    context.SetEnv('GYP_MSVS_VERSION', gyp_msvs_version)
    if os.path.exists(msvc):
      break
  else:
    # The break statement did not execute.
    raise Exception('Cannot find MSVC!')

  # Put MSVC in the path.
  vc = os.path.join(msvc, 'VC')
  comntools = os.path.join(msvc, 'Common7', 'Tools')
  perf = os.path.join(msvc, 'Team Tools', 'Performance Tools')
  context.SetEnv('PATH', os.pathsep.join([
      context.GetEnv('PATH'),
      vc,
      comntools,
      perf]))

  # SCons needs this variable to find vsvars.bat.
  # The end slash is needed because the batch files expect it.
  context.SetEnv(comntools_var, comntools + '\\')

  # This environment variable will SCons to print debug info while it searches
  # for MSVC.
  context.SetEnv('SCONS_MSCOMMON_DEBUG', '-')

  # Needed for finding devenv.
  context['msvc'] = msvc

  # The context on other systems has GYP_DEFINES set, set it for windows to be
  # able to save and restore without KeyError.
  context.SetEnv('GYP_DEFINES', '')


def SetupGypDefines(context, extra_vars=[]):
  context.SetEnv('GYP_DEFINES', ' '.join(context['gyp_vars'] + extra_vars))


def SetupLinuxEnvironment(context):
  SetupGypDefines(context, ['target_arch='+context['gyp_arch']])


def SetupMacEnvironment(context):
  SetupGypDefines(context)
  context.SetEnv('GYP_GENERATORS', 'ninja')


def SetupContextVars(context):
  # The branch is set to native_client on the main bots, on the trybots it's
  # set to ''.  Otherwise, we should assume a particular branch is being used.
  context['branch'] = os.environ.get('BUILDBOT_BRANCH', 'native_client')
  context['off_trunk'] = context['branch'] not in ['native_client', '']


def ValidatorTest(context, architecture, validator, warn_only=False):
  cmd = [
      sys.executable,
      'tests/abi_corpus/validator_regression_test.py',
      '--keep-going',
      '--validator', validator,
      '--arch', architecture
  ]
  if warn_only:
    cmd.append('--warn-only')
  Command(context, cmd=cmd)


def SummarizeCoverage(context):
  Command(context, [
      sys.executable,
      'tools/coverage_summary.py',
      context['platform'] + '-' + context['default_scons_platform'],
  ])


def ArchiveCoverage(context):
  gsutil = '/b/build/third_party/gsutil/gsutil'
  gsd_url = 'http://gsdview.appspot.com/nativeclient-coverage2/revs'
  variant_name = ('coverage-' + context['platform'] + '-' +
                  context['default_scons_platform'])
  coverage_path = variant_name + '/html/index.html'
  revision = os.environ.get('BUILDBOT_REVISION', 'None')
  link_url = gsd_url + '/' + revision + '/' + coverage_path
  gsd_base = 'gs://nativeclient-coverage2/revs'
  gs_path = gsd_base + '/' + revision + '/' + variant_name
  cov_dir = 'scons-out/' + variant_name + '/coverage'
  # Copy lcov file.
  Command(context, [
      sys.executable, gsutil,
      'cp', '-a', 'public-read',
      cov_dir + '/coverage.lcov',
      gs_path + '/coverage.lcov',
  ])
  # Copy html.
  Command(context, [
      sys.executable, gsutil,
      'cp', '-R', '-a', 'public-read',
      'html', gs_path,
  ], cwd=cov_dir)
  print '@@@STEP_LINK@view@%s@@@' % link_url

{'fraction_non_alphanumeric': 0.13185945519147257, 'fraction_numerical': 0.008093170153967627, 'mean_word_length': 3.6185050136736554, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '790344', 'n_tokens_mistral': 3499, 'n_tokens_neox': 3393, 'n_words': 843}