<commit_before>#!/bin/bash
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Main entry point for buildbots.
# For local testing set BUILDBOT_BUILDERNAME and TEST_BUILDBOT, e.g:
#  export TEST_BUILDBOT=1
#  export BUILDBOT_BUILDERNAME=linux-newlib-0
#  ./buildbot_selector.sh

set -o errexit

SCRIPT_DIR="$(cd $(dirname $0) && pwd)"
export NACL_SDK_ROOT="$(dirname ${SCRIPT_DIR})/out/nacl_sdk"

BOT_GSUTIL='/b/build/scripts/slave/gsutil'
if [ -e ${BOT_GSUTIL} ]; then
  export GSUTIL=${BOT_GSUTIL}
else
  export GSUTIL=gsutil
fi


# The bots set the BOTO_CONFIG environment variable to a different .boto file
# (currently /b/build/site-config/.boto). override this to the gsutil default
# which has access to gs://nativeclient-mirror.
# gsutil also looks for AWS_CREDENTIAL_FILE, so clear that too.
unset AWS_CREDENTIAL_FILE
unset BOTO_CONFIG

RESULT=0

# TODO: Eliminate this dependency if possible.
# This is required on OSX so that the naclports version of pkg-config can be
# found.
export PATH=${PATH}:/opt/local/bin

StartBuild() {
  cd ${BOT_DIR}
  export NACL_ARCH=$2

  echo "@@@BUILD_STEP $2 setup@@@"
  if ! ./$1 ; then
    RESULT=1
  fi
  cd -
}

Publish() {
  if [ -n "${NACLPORTS_NO_UPLOAD:-}" ]; then
    return
  fi
  echo "@@@BUILD_STEP upload binaries@@@"
  UPLOAD_PATH=nativeclient-mirror/naclports/${PEPPER_DIR}/
  UPLOAD_PATH+=${BUILDBOT_REVISION}/publish
  SRC_PATH=out/publish
  echo "Uploading to ${UPLOAD_PATH}"

  ${GSUTIL} cp -R -a public-read ${SRC_PATH} gs://${UPLOAD_PATH}

  URL="http://gsdview.appspot.com/${UPLOAD_PATH}"
  echo "@@@STEP_LINK@browse@${URL}@@@"
}

# Ignore 'periodic-' prefix.
BUILDBOT_BUILDERNAME=${BUILDBOT_BUILDERNAME#periodic-}

if [ "${BUILDBOT_BUILDERNAME}" != "linux-sdk" ]; then
  # Decode buildername.
  readonly BNAME_REGEX="(.+)-(.+)-(.+)"
  if [[ ${BUILDBOT_BUILDERNAME} =~ $BNAME_REGEX ]]; then
    readonly OS=${BASH_REMATCH[1]}
    readonly LIBC=${BASH_REMATCH[2]}
    readonly SHARD=${BASH_REMATCH[3]}
  else
    echo "Bad BUILDBOT_BUILDERNAME: ${BUILDBOT_BUILDERNAME}" 1>&2
    exit 1
  fi

  # Select platform specific things.
  if [ "$OS" = "mac" ]; then
    readonly PYTHON=python
    # Use linux config on mac too.
    readonly BOT_OS_DIR=linux
  elif [ "$OS" = "linux" ]; then
    readonly PYTHON=python
    readonly BOT_OS_DIR=linux
  elif [ "$OS" = "win" ]; then
    readonly PYTHON=python.bat
    readonly BOT_OS_DIR=windows
  else
    echo "Bad OS: ${OS}" 1>&2
    exit 1
  fi

  # Select libc
  if [ "$LIBC" = "glibc" ]; then
    export NACL_GLIBC=1
  elif [ "$LIBC" = "newlib" ]; then
    export NACL_GLIBC=0
  elif [ "$LIBC" = "pnacl_newlib" ]; then
    export NACL_GLIBC=0
  else
    echo "Bad LIBC: ${LIBC}" 1>&2
    exit 1
  fi
fi

# Goto src/
cd ${SCRIPT_DIR}/..

# Install SDK.
echo "@@@BUILD_STEP Install Latest SDK@@@"
if [ -z "${TEST_BUILDBOT:-}" -o ! -d ${NACL_SDK_ROOT} ]; then
  ${PYTHON} build_tools/download_sdk.py
fi

# PEPPER_DIR is the root direcotry name within the bundle. e.g. pepper_28
export PEPPER_VERSION=$(${NACL_SDK_ROOT}/tools/getos.py --sdk-version)
export PEPPER_DIR=pepper_${PEPPER_VERSION}

# This a temporary hack until the pnacl support is more mature
if [ ${LIBC} = "pnacl_newlib" ] ; then
  BOT_DIR=${SCRIPT_DIR}/bots
  StartBuild pnacl_bots.sh pnacl
  exit 0
fi

# The SDK builder builds a subset of the ports, but with multiple
# configurations.
if [ "${BUILDBOT_BUILDERNAME}" = "linux-sdk" ]; then
  cd ${SCRIPT_DIR}/bots/linux
  ./naclports-linux-sdk-bundle.sh
  exit 0
fi

# Compute script name.
readonly SCRIPT_NAME="naclports-${BOT_OS_DIR}-${SHARD}.sh"
BOT_DIR=${SCRIPT_DIR}/bots/${BOT_OS_DIR}

# Build 32-bit.
StartBuild ${SCRIPT_NAME} i686

# Build 64-bit.
StartBuild ${SCRIPT_NAME} x86_64

# Build ARM.
if [ ${NACL_GLIBC} != "1" ]; then
  StartBuild ${SCRIPT_NAME} arm
fi

Publish

exit $RESULT<commit_msg>Fixes for upload to commondatastorage.<commit_after>#!/bin/bash
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Main entry point for buildbots.
# For local testing set BUILDBOT_BUILDERNAME and TEST_BUILDBOT, e.g:
#  export TEST_BUILDBOT=1
#  export BUILDBOT_BUILDERNAME=linux-newlib-0
#  ./buildbot_selector.sh

set -o errexit

SCRIPT_DIR="$(cd $(dirname $0) && pwd)"
export NACL_SDK_ROOT="$(dirname ${SCRIPT_DIR})/out/nacl_sdk"

BOT_GSUTIL='/b/build/scripts/slave/gsutil'
if [ -e ${BOT_GSUTIL} ]; then
  export GSUTIL=${BOT_GSUTIL}
else
  export GSUTIL=gsutil
fi


# The bots set the BOTO_CONFIG environment variable to a different .boto file
# (currently /b/build/site-config/.boto). override this to the gsutil default
# which has access to gs://nativeclient-mirror.
# gsutil also looks for AWS_CREDENTIAL_FILE, so clear that too.
unset AWS_CREDENTIAL_FILE
unset BOTO_CONFIG

RESULT=0

# TODO: Eliminate this dependency if possible.
# This is required on OSX so that the naclports version of pkg-config can be
# found.
export PATH=${PATH}:/opt/local/bin

StartBuild() {
  cd ${BOT_DIR}
  export NACL_ARCH=$2

  echo "@@@BUILD_STEP $2 setup@@@"
  if ! ./$1 ; then
    RESULT=1
  fi
  cd -
}

Publish() {
  if [ -n "${NACLPORTS_NO_UPLOAD:-}" ]; then
    return
  fi
  echo "@@@BUILD_STEP upload binaries@@@"
  UPLOAD_PATH=nativeclient-mirror/naclports/${PEPPER_DIR}/
  UPLOAD_PATH+=${BUILDBOT_REVISION}/publish
  SRC_PATH=out/publish
  echo "Uploading to ${UPLOAD_PATH}"

  ${GSUTIL} cp -R -a public-read ${SRC_PATH}/* gs://${UPLOAD_PATH}/

  URL="http://gsdview.appspot.com/${UPLOAD_PATH}/"
  echo "@@@STEP_LINK@browse@${URL}@@@"
}

# Ignore 'periodic-' prefix.
BUILDBOT_BUILDERNAME=${BUILDBOT_BUILDERNAME#periodic-}

if [ "${BUILDBOT_BUILDERNAME}" != "linux-sdk" ]; then
  # Decode buildername.
  readonly BNAME_REGEX="(.+)-(.+)-(.+)"
  if [[ ${BUILDBOT_BUILDERNAME} =~ $BNAME_REGEX ]]; then
    readonly OS=${BASH_REMATCH[1]}
    readonly LIBC=${BASH_REMATCH[2]}
    readonly SHARD=${BASH_REMATCH[3]}
  else
    echo "Bad BUILDBOT_BUILDERNAME: ${BUILDBOT_BUILDERNAME}" 1>&2
    exit 1
  fi

  # Select platform specific things.
  if [ "$OS" = "mac" ]; then
    readonly PYTHON=python
    # Use linux config on mac too.
    readonly BOT_OS_DIR=linux
  elif [ "$OS" = "linux" ]; then
    readonly PYTHON=python
    readonly BOT_OS_DIR=linux
  elif [ "$OS" = "win" ]; then
    readonly PYTHON=python.bat
    readonly BOT_OS_DIR=windows
  else
    echo "Bad OS: ${OS}" 1>&2
    exit 1
  fi

  # Select libc
  if [ "$LIBC" = "glibc" ]; then
    export NACL_GLIBC=1
  elif [ "$LIBC" = "newlib" ]; then
    export NACL_GLIBC=0
  elif [ "$LIBC" = "pnacl_newlib" ]; then
    export NACL_GLIBC=0
  else
    echo "Bad LIBC: ${LIBC}" 1>&2
    exit 1
  fi
fi

# Goto src/
cd ${SCRIPT_DIR}/..

# Install SDK.
echo "@@@BUILD_STEP Install Latest SDK@@@"
if [ -z "${TEST_BUILDBOT:-}" -o ! -d ${NACL_SDK_ROOT} ]; then
  ${PYTHON} build_tools/download_sdk.py
fi

# PEPPER_DIR is the root direcotry name within the bundle. e.g. pepper_28
export PEPPER_VERSION=$(${NACL_SDK_ROOT}/tools/getos.py --sdk-version)
export PEPPER_DIR=pepper_${PEPPER_VERSION}

# The SDK builder builds a subset of the ports, but with multiple
# configurations.
if [ "${BUILDBOT_BUILDERNAME}" = "linux-sdk" ]; then
  cd ${SCRIPT_DIR}/bots/linux
  ./naclports-linux-sdk-bundle.sh
  exit 0
fi

# This a temporary hack until the pnacl support is more mature
if [ ${LIBC} = "pnacl_newlib" ] ; then
  BOT_DIR=${SCRIPT_DIR}/bots
  StartBuild pnacl_bots.sh pnacl
else
  # Compute script name.
  readonly SCRIPT_NAME="naclports-${BOT_OS_DIR}-${SHARD}.sh"
  BOT_DIR=${SCRIPT_DIR}/bots/${BOT_OS_DIR}

  # Build 32-bit.
  StartBuild ${SCRIPT_NAME} i686

  # Build 64-bit.
  StartBuild ${SCRIPT_NAME} x86_64

  # Build ARM.
  if [ ${NACL_GLIBC} != "1" ]; then
    StartBuild ${SCRIPT_NAME} arm
  fi
fi

Publish

exit $RESULT
{'fraction_non_alphanumeric': 0.13701560140915953, 'fraction_numerical': 0.010694514343231001, 'mean_word_length': 3.9712320200125077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2247221', 'n_tokens_mistral': 3211, 'n_tokens_neox': 3093, 'n_words': 780}