<commit_before>#!/usr/bin/env python
# Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""This script fetches and prepares an SDK chroot.
"""

import optparse
import os
import sys
import urlparse

from chromite.buildbot import constants
from chromite.lib import cgroups
from chromite.lib import cros_build_lib
from chromite.lib import locking
from chromite.lib import sudo

cros_build_lib.STRICT_SUDO = True


DEFAULT_URL = 'https://commondatastorage.googleapis.com/chromiumos-sdk/'
SDK_SUFFIXES = ['.tbz2', '.tar.xz']

SRC_ROOT = os.path.realpath(constants.SOURCE_ROOT)
SDK_DIR = os.path.join(SRC_ROOT, 'sdks')
OVERLAY_DIR = os.path.join(SRC_ROOT, 'src/third_party/chromiumos-overlay')
SDK_VERSION_FILE = os.path.join(OVERLAY_DIR,
                                'chromeos/binhost/host/sdk_version.conf')

# TODO(zbehan): Remove the dependency on these, reimplement them in python
MAKE_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/make_chroot.sh')]
ENTER_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/enter_chroot.sh')]

# We need these tools to run. Very common tools (tar,..) are ommited.
NEEDED_TOOLS = ['curl']

def GetHostArch():
  """Returns a string for the host architecture"""
  out = cros_build_lib.RunCommand(['uname', '-m'],
      redirect_stdout=True, print_cmd=False).output
  return out.rstrip('\n')

def CheckPrerequisites(needed_tools):
  """Verifies that the required tools are present on the system.

  This is especially important as this script is intended to run
  outside the chroot.

  Arguments:
    needed_tools: an array of string specified binaries to look for.

  Returns:
    True if all needed tools were found.
  """
  missing = []
  for tool in needed_tools:
    cmd = ['which', tool]
    try:
      cros_build_lib.RunCommand(cmd, print_cmd=False, redirect_stdout=True,
                                combine_stdout_stderr=True)
    except cros_build_lib.RunCommandError:
      missing.append(tool)
  return missing


def GetLatestVersion():
  """Extracts latest version from chromiumos-overlay."""
  sdk_file = open(SDK_VERSION_FILE)
  buf = sdk_file.readline().rstrip('\n').split('=')
  if buf[0] != 'SDK_LATEST_VERSION':
    raise Exception('Malformed version file')
  return buf[1].strip('"')


def GetArchStageTarballs(tarballArch, version):
  """Returns the URL for a given arch/version"""
  D = { 'x86_64': 'cros-sdk-' }
  try:
    return [DEFAULT_URL + D[tarballArch] + version + x for x in SDK_SUFFIXES]
  except KeyError:
    raise SystemExit('Unsupported arch: %s' % (tarballArch,))


def FetchRemoteTarballs(urls):
  """Fetches a tarball given by url, and place it in sdk/.

  Args:
    urls: List of URLs to try to download. Download will stop on first success.

  Returns:
    Full path to the downloaded file
  """

  def RunCurl(args, **kwargs):
    """Runs curl and wraps around all necessary hacks."""
    cmd = ['curl']
    cmd.extend(args)

    # These values were discerned via scraping the curl manpage; they're all
    # retry related (dns failed, timeout occurred, etc, see  the manpage for
    # exact specifics of each).
    # Note we allow 22 to deal w/ 500's- they're thrown by google storage
    # occasionally.
    # Finally, we do not use curl's --retry option since it generally doesn't
    # actually retry anything; code 18 for example, it will not retry on.
    retriable_exits = frozenset([5, 6, 7, 15, 18, 22, 26, 28, 52])
    try:
      return cros_build_lib.RunCommandWithRetries(
          5, cmd, sleep=3, retry_on=retriable_exits, **kwargs)
    except cros_build_lib.RunCommandError, e:
      code = e.result.returncode
      if e.returncode in (51, 58, 60):
        # These are the return codes of failing certs as per 'man curl'.<commit_msg>Fix AttributeError accessing the return code.<commit_after>#!/usr/bin/env python
# Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""This script fetches and prepares an SDK chroot.
"""

import optparse
import os
import sys
import urlparse

from chromite.buildbot import constants
from chromite.lib import cgroups
from chromite.lib import cros_build_lib
from chromite.lib import locking
from chromite.lib import sudo

cros_build_lib.STRICT_SUDO = True


DEFAULT_URL = 'https://commondatastorage.googleapis.com/chromiumos-sdk/'
SDK_SUFFIXES = ['.tbz2', '.tar.xz']

SRC_ROOT = os.path.realpath(constants.SOURCE_ROOT)
SDK_DIR = os.path.join(SRC_ROOT, 'sdks')
OVERLAY_DIR = os.path.join(SRC_ROOT, 'src/third_party/chromiumos-overlay')
SDK_VERSION_FILE = os.path.join(OVERLAY_DIR,
                                'chromeos/binhost/host/sdk_version.conf')

# TODO(zbehan): Remove the dependency on these, reimplement them in python
MAKE_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/make_chroot.sh')]
ENTER_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/enter_chroot.sh')]

# We need these tools to run. Very common tools (tar,..) are ommited.
NEEDED_TOOLS = ['curl']

def GetHostArch():
  """Returns a string for the host architecture"""
  out = cros_build_lib.RunCommand(['uname', '-m'],
      redirect_stdout=True, print_cmd=False).output
  return out.rstrip('\n')

def CheckPrerequisites(needed_tools):
  """Verifies that the required tools are present on the system.

  This is especially important as this script is intended to run
  outside the chroot.

  Arguments:
    needed_tools: an array of string specified binaries to look for.

  Returns:
    True if all needed tools were found.
  """
  missing = []
  for tool in needed_tools:
    cmd = ['which', tool]
    try:
      cros_build_lib.RunCommand(cmd, print_cmd=False, redirect_stdout=True,
                                combine_stdout_stderr=True)
    except cros_build_lib.RunCommandError:
      missing.append(tool)
  return missing


def GetLatestVersion():
  """Extracts latest version from chromiumos-overlay."""
  sdk_file = open(SDK_VERSION_FILE)
  buf = sdk_file.readline().rstrip('\n').split('=')
  if buf[0] != 'SDK_LATEST_VERSION':
    raise Exception('Malformed version file')
  return buf[1].strip('"')


def GetArchStageTarballs(tarballArch, version):
  """Returns the URL for a given arch/version"""
  D = { 'x86_64': 'cros-sdk-' }
  try:
    return [DEFAULT_URL + D[tarballArch] + version + x for x in SDK_SUFFIXES]
  except KeyError:
    raise SystemExit('Unsupported arch: %s' % (tarballArch,))


def FetchRemoteTarballs(urls):
  """Fetches a tarball given by url, and place it in sdk/.

  Args:
    urls: List of URLs to try to download. Download will stop on first success.

  Returns:
    Full path to the downloaded file
  """

  def RunCurl(args, **kwargs):
    """Runs curl and wraps around all necessary hacks."""
    cmd = ['curl']
    cmd.extend(args)

    # These values were discerned via scraping the curl manpage; they're all
    # retry related (dns failed, timeout occurred, etc, see  the manpage for
    # exact specifics of each).
    # Note we allow 22 to deal w/ 500's- they're thrown by google storage
    # occasionally.
    # Finally, we do not use curl's --retry option since it generally doesn't
    # actually retry anything; code 18 for example, it will not retry on.
    retriable_exits = frozenset([5, 6, 7, 15, 18, 22, 26, 28, 52, 56])
    try:
      return cros_build_lib.RunCommandWithRetries(
          5, cmd, sleep=3, retry_on=retriable_exits, **kwargs)
    except cros_build_lib.RunCommandError, e:
      code = e.result.returncode
      if code in (51, 58, 60):
        # These are the return codes of failing certs as per 'man curl'.
{'fraction_non_alphanumeric': 0.10487616099071208, 'fraction_numerical': 0.011867905056759546, 'mean_word_length': 3.6789378394689196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3784412', 'n_tokens_mistral': 2623, 'n_tokens_neox': 2445, 'n_words': 801}