<commit_before>#!/bin/bash
#
# Copyright (c) 2020 The Orbit Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Fail on any error.
set -e

if [ -z "${CONAN_PROFILE}" ]; then
  CONAN_PROFILE="$(basename $(dirname "${KOKORO_JOB_NAME}"))"

  # That's a temporary work-around. As long as the kokoro jobs don't carry the
  # conan profile in its name this mapping will do the trick.
  declare -A profile_mapping=( \
    [gcp_ubuntu]="clang7_relwithdebinfo" \
    [gcp_ubuntu_ggp]="ggp_relwithdebinfo" \
    [gcp_ubuntu_test]="clang7_relwithdebinfo" \
    [gcp_windows]="msvc2017_relwithdebinfo" \
    [gcp_windows_test]="msvc2017_relwithdebinfo" \
  )

  CONAN_PROFILE="${profile_mapping[${CONAN_PROFILE}]-${CONAN_PROFILE}}"
fi

if [ -z "${CONAN_PROFILE}" ]; then
  echo -n "No conan profile was set nor one could be auto-detected. "
  echo "Aborting this build!"
  exit 2
fi

source $(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd)/upload_symbols.sh

if [ -n "$1" ]; then
  # We are inside the docker container

  readonly REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../" >/dev/null 2>&1 && pwd )"
  readonly MOUNT_POINT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../../../" >/dev/null 2>&1 && pwd )"
  readonly KEYSTORE_PATH="${MOUNT_POINT}/keystore"

  export QT_QPA_PLATFORM=offscreen

  if [ -z "$BUILD_TYPE" ]; then
    readonly BUILD_TYPE="$(basename "$KOKORO_JOB_NAME")"
  fi

  function cleanup {
    # Delete all unnecessary files from the src/-directory.
    # Kokoro would copy them otherwise before applying the artifacts regex

    set +e # This is allowed to fail when deleting
    if [ "${BUILD_TYPE}" == "presubmit" ]; then
      # In the presubmit case we only spare the test-results and this script.
      find "$MOUNT_POINT" -depth -mindepth 1 | \
        grep -v 'orbitprofiler/kokoro' | \
        grep -v 'testresults/' | \
        while read file; do
          if [[ -d $file ]]; then
            # That might give an error message when the directory is not empty.
            # That's okay.
            rmdir --ignore-fail-on-non-empty "$file"
          elif [[ -e $file ]]; then
            rm "$file"
          fi
        done
    else
      # In the non-presubmit case we spare the whole build dir and this script.
      find "$MOUNT_POINT" -depth -mindepth 1 | \
        grep -v 'orbitprofiler/kokoro' | \
        grep -v 'orbitprofiler/build' | \
        while read file; do
          if [[ -d $file ]]; then
            # That might give an error message when the directory is not empty.
            # That's okay.
            rmdir --ignore-fail-on-non-empty "$file"
          elif [[ -e $file ]]; then
            rm "$file"
          fi
        done
    fi
  }
  trap cleanup EXIT

  echo "Using conan profile ${CONAN_PROFILE} and performing a ${BUILD_TYPE} build."

  set +e
  if [ "$BUILD_TYPE" = "release" ] \
     && [ -z "${ORBIT_BYPASS_RELEASE_CHECK}" ] \
     && ! git -C "${REPO_ROOT}" describe --tags --exact-match > /dev/null; then
    echo -n "We are currently conducting a release build, but we aren't on a tag."
    echo    " Aborting the build..."
    echo -n "Maybe you missed pushing the release version tag?"
    echo    " Please consult the release playbook for advice."
    exit 1
  fi
  set -e

  # That's a temporary solution. The docker containers should have the
  # correct version of conan already preinstalled. This step will be removed
  # when the docker containers are restructured and versioned.
  pip3 install conan==1.27.1

  echo "Installing conan configuration (profiles, settings, etc.)..."
  ${REPO_ROOT}/third_party/conan/configs/install.sh


  if [ "$(uname -s)" == "Linux" ]; then
    readonly OS="linux"
  else
    readonly OS="windows"
  fi

  CRASHDUMP_SERVER=""
  readonly CRASH_SERVER_URL_FILE="${KEYSTORE_PATH}/74938_orbitprofiler_crashdump_collection_server"
  if [ -f "$CRASH_SERVER_URL_FILE" ]; then
    CRASHDUMP_SERVER="$(cat "${CRASH_SERVER_URL_FILE}" | tr -d '\n')"
  fi

  # Building Orbit
  mkdir -p "${REPO_ROOT}/build/"

  # Patching the lockfile is a temporary work-around. As soon as we move to
  # conan-1.28's base-lockfiles, this will be gone.
  cp -v "${REPO_ROOT}/third_party/conan/lockfiles/${OS}/${CONAN_PROFILE}/conan.lock" \
        "${REPO_ROOT}/build/conan.lock"
  sed -i -e "s|crashdump_server=|crashdump_server=$CRASHDUMP_SERVER|" \
            "${REPO_ROOT}/build/conan.lock"

  if [[ $CONAN_PROFILE == ggp_* ]]; then
    readonly PACKAGING_OPTION="-o debian_packaging=True"
    sed -i -e "s/debian_packaging=False/debian_packaging=True/" \
      "${REPO_ROOT}/build/conan.lock"
  else
    readonly PACKAGING_OPTION=""
  fi

  conan install -u -pr ${CONAN_PROFILE} -if "${REPO_ROOT}/build/" \
          --build outdated \
          -o crashdump_server="$CRASHDUMP_SERVER" \
          $PACKAGING_OPTION \
          --lockfile="${REPO_ROOT}/build/conan.lock" \
          "${REPO_ROOT}"
  conan build -bf "${REPO_ROOT}/build/" "${REPO_ROOT}"
  conan package -bf "${REPO_ROOT}/build/" "${REPO_ROOT}"

  # Uploading symbols to the symbol server
  if [ "${BUILD_TYPE}" == "release" ] \
     || [ "${BUILD_TYPE}" == "nightly" ] \
     || [ "${BUILD_TYPE}" == "continuous_on_release_branch" ]; then
    set +e
    api_key=$(get_api_key "${OAUTH_TOKEN_HEADER}")
    upload_debug_symbols "${api_key}" "${REPO_ROOT}/build/bin"
    set -e
  fi

  # Signing the debian package
  if [ -f "${KEYSTORE_PATH}/74938_SigningPrivateGpg" ] && [[ $CONAN_PROFILE == ggp_* ]]; then
    rm -rf ~/.gnupg/
    rm -rf /dev/shm/signing.gpg
    mkdir -p ~/.gnupg
    chmod 700 ~/.gnupg
    echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf

    GPG_OPTIONS="--pinentry-mode loopback --batch --no-tty --yes --no-default-keyring --keyring /dev/shm/signing.gpg --passphrase-file ${KEYSTORE_PATH}/74938_SigningPrivateGpgKeyPassword"

    gpg ${GPG_OPTIONS} --import ${KEYSTORE_PATH}/74938_SigningPrivateGpg
<commit_msg>Use base-lockfile in CI builds<commit_after>#!/bin/bash
#
# Copyright (c) 2020 The Orbit Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Fail on any error.
set -e

if [ -z "${CONAN_PROFILE}" ]; then
  CONAN_PROFILE="$(basename $(dirname "${KOKORO_JOB_NAME}"))"

  # That's a temporary work-around. As long as the kokoro jobs don't carry the
  # conan profile in its name this mapping will do the trick.
  declare -A profile_mapping=( \
    [gcp_ubuntu]="clang7_relwithdebinfo" \
    [gcp_ubuntu_ggp]="ggp_relwithdebinfo" \
    [gcp_ubuntu_test]="clang7_relwithdebinfo" \
    [gcp_windows]="msvc2017_relwithdebinfo" \
    [gcp_windows_test]="msvc2017_relwithdebinfo" \
  )

  CONAN_PROFILE="${profile_mapping[${CONAN_PROFILE}]-${CONAN_PROFILE}}"
fi

if [ -z "${CONAN_PROFILE}" ]; then
  echo -n "No conan profile was set nor one could be auto-detected. "
  echo "Aborting this build!"
  exit 2
fi

source $(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd)/upload_symbols.sh

if [ -n "$1" ]; then
  # We are inside the docker container

  readonly REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../" >/dev/null 2>&1 && pwd )"
  readonly MOUNT_POINT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../../../" >/dev/null 2>&1 && pwd )"
  readonly KEYSTORE_PATH="${MOUNT_POINT}/keystore"

  export QT_QPA_PLATFORM=offscreen

  if [ -z "$BUILD_TYPE" ]; then
    readonly BUILD_TYPE="$(basename "$KOKORO_JOB_NAME")"
  fi

  function cleanup {
    # Delete all unnecessary files from the src/-directory.
    # Kokoro would copy them otherwise before applying the artifacts regex

    set +e # This is allowed to fail when deleting
    if [ "${BUILD_TYPE}" == "presubmit" ]; then
      # In the presubmit case we only spare the test-results and this script.
      find "$MOUNT_POINT" -depth -mindepth 1 | \
        grep -v 'orbitprofiler/kokoro' | \
        grep -v 'testresults/' | \
        while read file; do
          if [[ -d $file ]]; then
            # That might give an error message when the directory is not empty.
            # That's okay.
            rmdir --ignore-fail-on-non-empty "$file"
          elif [[ -e $file ]]; then
            rm "$file"
          fi
        done
    else
      # In the non-presubmit case we spare the whole build dir and this script.
      find "$MOUNT_POINT" -depth -mindepth 1 | \
        grep -v 'orbitprofiler/kokoro' | \
        grep -v 'orbitprofiler/build' | \
        while read file; do
          if [[ -d $file ]]; then
            # That might give an error message when the directory is not empty.
            # That's okay.
            rmdir --ignore-fail-on-non-empty "$file"
          elif [[ -e $file ]]; then
            rm "$file"
          fi
        done
    fi
  }
  trap cleanup EXIT

  echo "Using conan profile ${CONAN_PROFILE} and performing a ${BUILD_TYPE} build."

  set +e
  if [ "$BUILD_TYPE" = "release" ] \
     && [ -z "${ORBIT_BYPASS_RELEASE_CHECK}" ] \
     && ! git -C "${REPO_ROOT}" describe --tags --exact-match > /dev/null; then
    echo -n "We are currently conducting a release build, but we aren't on a tag."
    echo    " Aborting the build..."
    echo -n "Maybe you missed pushing the release version tag?"
    echo    " Please consult the release playbook for advice."
    exit 1
  fi
  set -e

  # That's a temporary solution. The docker containers should have the
  # correct version of conan already preinstalled. This step will be removed
  # when the docker containers are restructured and versioned.
  pip3 install conan==1.29.2

  echo "Installing conan configuration (profiles, settings, etc.)..."
  ${REPO_ROOT}/third_party/conan/configs/install.sh


  if [ "$(uname -s)" == "Linux" ]; then
    readonly OS="linux"
  else
    readonly OS="windows"
  fi

  CRASHDUMP_SERVER=""
  readonly CRASH_SERVER_URL_FILE="${KEYSTORE_PATH}/74938_orbitprofiler_crashdump_collection_server"
  if [ -f "$CRASH_SERVER_URL_FILE" ]; then
    CRASHDUMP_SERVER="$(cat "${CRASH_SERVER_URL_FILE}" | tr -d '\n')"
  fi

  # Building Orbit
  mkdir -p "${REPO_ROOT}/build/"

  if [[ $CONAN_PROFILE == ggp_* ]]; then
    readonly PACKAGING_OPTION="-o debian_packaging=True"
  else
    readonly PACKAGING_OPTION=""
  fi

  conan lock create "${REPO_ROOT}/conanfile.py" --user=orbitdeps --channel=stable \
    --lockfile="${REPO_ROOT}/third_party/conan/lockfiles/base.lock" -u -pr ${CONAN_PROFILE} \
    -o crashdump_server="$CRASHDUMP_SERVER" $PACKAGING_OPTION \
    --lockfile-out="${REPO_ROOT}/build/conan.lock" || exit $?

  conan install -if "${REPO_ROOT}/build/" \
          --build outdated \
          --lockfile="${REPO_ROOT}/build/conan.lock" \
          "${REPO_ROOT}"
  conan build -bf "${REPO_ROOT}/build/" "${REPO_ROOT}"
  conan package -bf "${REPO_ROOT}/build/" "${REPO_ROOT}"

  # Uploading symbols to the symbol server
  if [ "${BUILD_TYPE}" == "release" ] \
     || [ "${BUILD_TYPE}" == "nightly" ] \
     || [ "${BUILD_TYPE}" == "continuous_on_release_branch" ]; then
    set +e
    api_key=$(get_api_key "${OAUTH_TOKEN_HEADER}")
    upload_debug_symbols "${api_key}" "${REPO_ROOT}/build/bin"
    set -e
  fi

  # Signing the debian package
  if [ -f "${KEYSTORE_PATH}/74938_SigningPrivateGpg" ] && [[ $CONAN_PROFILE == ggp_* ]]; then
    rm -rf ~/.gnupg/
    rm -rf /dev/shm/signing.gpg
    mkdir -p ~/.gnupg
    chmod 700 ~/.gnupg
    echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf

    GPG_OPTIONS="--pinentry-mode loopback --batch --no-tty --yes --no-default-keyring --keyring /dev/shm/signing.gpg --passphrase-file ${KEYSTORE_PATH}/74938_SigningPrivateGpgKeyPassword"

    gpg ${GPG_OPTIONS} --import ${KEYSTORE_PATH}/74938_SigningPrivateGpg

{'fraction_non_alphanumeric': 0.14060637029466871, 'fraction_numerical': 0.009793902231306422, 'mean_word_length': 2.8590207032533685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1034954', 'n_tokens_mistral': 4174, 'n_tokens_neox': 3974, 'n_words': 1102}