<commit_before>#!/usr/bin/python

# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import errno
import optparse
import os
import shutil
import subprocess
import sys

from cbuildbot_config import config

_DEFAULT_RETRIES=3

# Utility functions

def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
               exit_code=False, redirect_stdout=False, redirect_stderr=False,
               cwd=None, input=None):
  # Print out the command before running.
  if print_cmd:
    print >> sys.stderr, "CBUILDBOT -- RunCommand:", ' '.join(cmd)
  if redirect_stdout:
    stdout = subprocess.PIPE
  else:
    stdout = None
  if redirect_stderr:
    stderr = subprocess.PIPE
  else:
    stderr = None
  if input:
    stdin = subprocess.PIPE
  else:
      stdin = None
  proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
                          stdout=stdout, stderr=stderr)
  (output, error) = proc.communicate(input)
  if exit_code:
    return proc.returncode
  if not error_ok and proc.returncode != 0:
    raise Exception('Command "%s" failed.\n' % (' '.join(cmd)) +
                    (error_message or error or output or ''))
  return output

def MakeDir(path, parents=False):
  try:
    os.makedirs(path)
  except OSError, e:
    if e.errno == errno.EEXIST and parents:
      pass
    else:
      raise

def RepoSync(buildroot, rw_checkout, retries=_DEFAULT_RETRIES):
  while retries > 0:
    try:
      RunCommand(['repo', 'sync'], cwd=buildroot)
      if rw_checkout:
        # Always re-run in case of new git repos or repo sync
        # failed in a previous run because of a forced Stop Build.
        RunCommand(['repo', 'forall', '-c', 'git', 'config',
                    'url.ssh://<EMAIL>@gitrw.chromium.org:9222.pushinsteadof',
                    'http://src.chromium.org/git'], cwd=buildroot)
      retries = 0
    except:
      retries -= 1
      if retries > 0:
        print >>sys.stderr, 'CBUILDBOT -- Repo Sync Failed, retrying'
      else:
        print >>sys.stderr, 'CBUILDBOT -- Retries exhausted'
        raise

# Main functions

def _FullCheckout(buildroot, rw_checkout=True, retries=_DEFAULT_RETRIES):
  RunCommand(['sudo', 'rm', '-rf', buildroot])
  MakeDir(buildroot, parents=True)
  RunCommand(['repo', 'init', '-u', 'http://src.chromium.org/git/manifest'],
             cwd=buildroot, input='\n\ny\n')
  RepoSync(buildroot, rw_checkout, retries)

def _IncrementalCheckout(buildroot, rw_checkout=True,
                         retries=_DEFAULT_RETRIES):
  RepoSync(buildroot, rw_checkout, retries)

def _MakeChroot(buildroot):
  cwd = os.path.join(buildroot, 'src', 'scripts')
  RunCommand(['./make_chroot', '--fast'], cwd=cwd)

def _SetupBoard(buildroot, board='x86-generic'):
  cwd = os.path.join(buildroot, 'src', 'scripts')
  RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board],
             cwd=cwd)

def _Build(buildroot):
  cwd = os.path.join(buildroot, 'src', 'scripts')
  RunCommand(['./build_packages'], cwd=cwd)

def _UprevAllPackages(buildroot):
  cwd = os.path.join(buildroot, 'src', 'scripts')
  RunCommand(['./enter_chroot.sh', '--', './cros_mark_all_as_stable',
              '--tracking_branch="cros/master"'],
             cwd=cwd)

def _UprevPackages(buildroot, revisionfile):
  revisions = None
  if (revision_file):
    rev_file = revisionfile.open(revisionfile)
    revisions = rev_file.read()
    rev_file.close()

  # Note:  Revisions == "None" indicates a Force Build.
  if revisions and revisions != 'None':
    print 'CBUILDBOT - Revision list found %s' % revisions
    print 'Revision list not yet propagating to build, marking all instead'

  _UprevAllPackages(buildroot)

def _UprevCleanup(buildroot):<commit_msg>cbuildbot: fix typo<commit_after>#!/usr/bin/python

# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import errno
import optparse
import os
import shutil
import subprocess
import sys

from cbuildbot_config import config

_DEFAULT_RETRIES=3

# Utility functions

def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
               exit_code=False, redirect_stdout=False, redirect_stderr=False,
               cwd=None, input=None):
  # Print out the command before running.
  if print_cmd:
    print >> sys.stderr, "CBUILDBOT -- RunCommand:", ' '.join(cmd)
  if redirect_stdout:
    stdout = subprocess.PIPE
  else:
    stdout = None
  if redirect_stderr:
    stderr = subprocess.PIPE
  else:
    stderr = None
  if input:
    stdin = subprocess.PIPE
  else:
      stdin = None
  proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
                          stdout=stdout, stderr=stderr)
  (output, error) = proc.communicate(input)
  if exit_code:
    return proc.returncode
  if not error_ok and proc.returncode != 0:
    raise Exception('Command "%s" failed.\n' % (' '.join(cmd)) +
                    (error_message or error or output or ''))
  return output

def MakeDir(path, parents=False):
  try:
    os.makedirs(path)
  except OSError, e:
    if e.errno == errno.EEXIST and parents:
      pass
    else:
      raise

def RepoSync(buildroot, rw_checkout, retries=_DEFAULT_RETRIES):
  while retries > 0:
    try:
      RunCommand(['repo', 'sync'], cwd=buildroot)
      if rw_checkout:
        # Always re-run in case of new git repos or repo sync
        # failed in a previous run because of a forced Stop Build.
        RunCommand(['repo', 'forall', '-c', 'git', 'config',
                    'url.ssh://<EMAIL>@gitrw.<EMAIL>:9222.pushinsteadof',
                    'http://src.chromium.org/git'], cwd=buildroot)
      retries = 0
    except:
      retries -= 1
      if retries > 0:
        print >>sys.stderr, 'CBUILDBOT -- Repo Sync Failed, retrying'
      else:
        print >>sys.stderr, 'CBUILDBOT -- Retries exhausted'
        raise

# Main functions

def _FullCheckout(buildroot, rw_checkout=True, retries=_DEFAULT_RETRIES):
  RunCommand(['sudo', 'rm', '-rf', buildroot])
  MakeDir(buildroot, parents=True)
  RunCommand(['repo', 'init', '-u', 'http://src.chromium.org/git/manifest'],
             cwd=buildroot, input='\n\ny\n')
  RepoSync(buildroot, rw_checkout, retries)

def _IncrementalCheckout(buildroot, rw_checkout=True,
                         retries=_DEFAULT_RETRIES):
  RepoSync(buildroot, rw_checkout, retries)

def _MakeChroot(buildroot):
  cwd = os.path.join(buildroot, 'src', 'scripts')
  RunCommand(['./make_chroot', '--fast'], cwd=cwd)

def _SetupBoard(buildroot, board='x86-generic'):
  cwd = os.path.join(buildroot, 'src', 'scripts')
  RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board],
             cwd=cwd)

def _Build(buildroot):
  cwd = os.path.join(buildroot, 'src', 'scripts')
  RunCommand(['./build_packages'], cwd=cwd)

def _UprevAllPackages(buildroot):
  cwd = os.path.join(buildroot, 'src', 'scripts')
  RunCommand(['./enter_chroot.sh', '--', './cros_mark_all_as_stable',
              '--tracking_branch="cros/master"'],
             cwd=cwd)

def _UprevPackages(buildroot, revisionfile):
  revisions = None
  if (revisionfile):
    rev_file = revisionfile.open(revisionfile)
    revisions = rev_file.read()
    rev_file.close()

  # Note:  Revisions == "None" indicates a Force Build.
  if revisions and revisions != 'None':
    print 'CBUILDBOT - Revision list found %s' % revisions
    print 'Revision list not yet propagating to build, marking all instead'

  _UprevAllPackages(buildroot)

def _UprevCleanup(buildroot):
{'fraction_non_alphanumeric': 0.1278038601982264, 'fraction_numerical': 0.004173187271778821, 'mean_word_length': 3.1907103825136613, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7009822', 'n_tokens_mistral': 2631, 'n_tokens_neox': 2537, 'n_words': 639}