<commit_before>#!/bin/bash
# Configure GCE instance to run the SecureDrop staging environment,
# including configuration tests. Test results will be collected as XML
# for storage as artifacts on the build, so devs can review via web.
set -e
set -u


TOPLEVEL="$(git rev-parse --show-toplevel)"
# shellcheck source=devops/gce-nested/ci-env.sh
. "${TOPLEVEL}/devops/gce-nested/ci-env.sh"

REMOTE_IP="$(gcloud_call compute instances describe \
            "${JOB_NAME}-${BUILD_NUM}" \
            --format="value(networkInterfaces[0].accessConfigs.natIP)")"
SSH_TARGET="${SSH_USER_NAME}@${REMOTE_IP}"
SSH_OPTS=(-i "$SSH_PRIVKEY" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null")

# Wrapper utility to run commands on remote GCE instance
function ssh_gce {
    # We want all args to be evaluated locally, then passed to the remote
    # host for execution, so we can safely disable shellcheck 2029.
    # shellcheck disable=SC2029
    ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "cd ~/securedrop-source/ && $*"
}

# Retrieve XML from test results, for posting as build artifact in CI.
function fetch_junit_test_results() {
    local remote_src
    local local_dest
    remote_src='junit/*xml'
    local_dest='junit/'
    scp "${SSH_OPTS[@]}" "${SSH_TARGET}:~/securedrop-source/${remote_src}" "$local_dest"
}

# Copy up securedrop repo to remote server
function copy_securedrop_repo() {
  rsync -a -e "ssh ${SSH_OPTS[*]}" \
      --exclude admin/.tox \
      --exclude '*.box' \
      --exclude '*.deb' \
      --exclude '*.pyc' \
      --exclude '*.venv' \
      --exclude .python3 \
      --exclude .mypy_cache \
      --exclude securedrop/.sass-cache \
      --exclude .gce.creds \
      --exclude '*.creds' \
      "${TOPLEVEL}/" "${SSH_TARGET}:~/securedrop-source"
}

# Main logic
copy_securedrop_repo
ssh_gce "make build-debs-notest"

# The test results should be collected regardless of pass/fail,
# so register a trap to ensure the fetch always runs.
trap fetch_junit_test_results EXIT

ssh_gce "make staging"
<commit_msg>Use Python 2 in "make staging" step of staging-test-with-rebase<commit_after>#!/bin/bash
# Configure GCE instance to run the SecureDrop staging environment,
# including configuration tests. Test results will be collected as XML
# for storage as artifacts on the build, so devs can review via web.
set -e
set -u


TOPLEVEL="$(git rev-parse --show-toplevel)"
# shellcheck source=devops/gce-nested/ci-env.sh
. "${TOPLEVEL}/devops/gce-nested/ci-env.sh"

REMOTE_IP="$(gcloud_call compute instances describe \
            "${JOB_NAME}-${BUILD_NUM}" \
            --format="value(networkInterfaces[0].accessConfigs.natIP)")"
SSH_TARGET="${SSH_USER_NAME}@${REMOTE_IP}"
SSH_OPTS=(-i "$SSH_PRIVKEY" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null")

# Wrapper utility to run commands on remote GCE instance
function ssh_gce {
    # We want all args to be evaluated locally, then passed to the remote
    # host for execution, so we can safely disable shellcheck 2029.
    # shellcheck disable=SC2029
    ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "cd ~/securedrop-source/ && $*"
}

# Retrieve XML from test results, for posting as build artifact in CI.
function fetch_junit_test_results() {
    local remote_src
    local local_dest
    remote_src='junit/*xml'
    local_dest='junit/'
    scp "${SSH_OPTS[@]}" "${SSH_TARGET}:~/securedrop-source/${remote_src}" "$local_dest"
}

# Copy up securedrop repo to remote server
function copy_securedrop_repo() {
  rsync -a -e "ssh ${SSH_OPTS[*]}" \
      --exclude admin/.tox \
      --exclude '*.box' \
      --exclude '*.deb' \
      --exclude '*.pyc' \
      --exclude '*.venv' \
      --exclude .python3 \
      --exclude .mypy_cache \
      --exclude securedrop/.sass-cache \
      --exclude .gce.creds \
      --exclude '*.creds' \
      "${TOPLEVEL}/" "${SSH_TARGET}:~/securedrop-source"
}

# Main logic
copy_securedrop_repo
ssh_gce "make build-debs-notest"

# The test results should be collected regardless of pass/fail,
# so register a trap to ensure the fetch always runs.
trap fetch_junit_test_results EXIT

ssh_gce "PYTHON_VERSION=2 make staging"

{'fraction_non_alphanumeric': 0.1336238198983297, 'fraction_numerical': 0.00532558702493343, 'mean_word_length': 3.447793326157158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2739093', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1358, 'n_words': 385}