<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: "End to End - Latest"
on:
  push:
    # run on pushes to master or release/*
    branches:
      - master
      - release/*
      - e2e*
      - loadgen_integration # will remove before merging
  workflow_dispatch:
    # trigger through UI or API
jobs:
  end-to-end-latest:
    runs-on: [self-hosted, e2e-worker]
    steps:
    - uses: actions/checkout@v2
    - name: Clean Project State
      timeout-minutes: 30
      run: |
        # clean GCP resources
        .github/workflows/e2e_scripts/clean.sh
    - name: Build Sandbox Services
      run: |
        set -x
        skaffold config set --global local-cluster false
        # tag with git hash
        skaffold build --default-repo=gcr.io/$PROJECT_ID/tests \
                       --tag=$GITHUB_SHA
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Build Load Generator
      run: |
        set -x
        gcloud auth configure-docker --quiet
        # push latest load generator to test image repository
        IMAGE_PATH=gcr.io/$PROJECT_ID/gcr.io/stackdriver-sandbox-230822/sandbox/loadgenerator/basic:$GITHUB_SHA
        docker build -t $IMAGE_PATH ./src/loadgenerator
        docker push $IMAGE_PATH
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Rewrite Manifests
      run: |
        set -x
        OLD_PATH=gcr.io/stackdriver-sandbox-230822
        NEW_PATH=gcr.io/stackdriver-sandbox-e2e/tests/$OLD_PATH
        # use newly built containers instead of latest pushed
        find "./kubernetes-manifests" -name '*.yaml' -exec sed -i -e "s#$OLD_PATH#$NEW_PATH#g" {} \;
        find "./kubernetes-manifests" -name '*.yaml' -exec sed -i -e "s#:latest#:$GITHUB_SHA#g" {} \;
    - name: Run Install Script
      timeout-minutes: 30
      run: |
        set -x
        # build cloud shell image
        docker build -t test-cloud-shell:$GITHUB_SHA ./cloud-shell
        # run install script
        docker run --rm \
          -e project_id=$PROJECT_ID \
          -e skip_workspace_prompt=1 \
          -e service_wait=1 \
          -v ~/.config:/root/.config \
          -v `pwd`:/sandbox-shared \
          --entrypoint /sandbox-shared/.github/workflows/e2e_scripts/run_install.sh \
          test-cloud-shell:$GITHUB_SHA
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Test Using Existing Project
      timeout-minutes: 30
      run: |
        set -x
        # run install script
        docker run --rm \
          -e project_id=$PROJECT_ID \
          -e skip_workspace_prompt=1 \
          -e service_wait=1 \
          -v ~/.config:/root/.config \
          -v `pwd`:/sandbox-shared \
          --entrypoint /sandbox-shared/.github/workflows/e2e_scripts/run_install.sh \
          test-cloud-shell:$GITHUB_SHA
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Run Provisioning Test
      timeout-minutes: 30
      run: |
        set -x
        # get cluster zones
        CLUSTER_ZONE=$(gcloud container clusters list --filter="name:cloud-ops-sandbox" --project $PROJECT_ID --format="value(zone)")
        LOADGEN_ZONE=$(gcloud container clusters list --filter="name:loadgenerator" --project $PROJECT_ID --format="value(zone)")
        # build provisioning test image
        docker build -t test-provisioning:$GITHUB_SHA tests/provisioning/.
        # run provisioning tests
        docker run --rm \
          -e GOOGLE_CLOUD_PROJECT=$PROJECT_ID \
          -e ZONE=$CLUSTER_ZONE \
          -e LOADGEN_ZONE=$LOADGEN_ZONE \
          -v ~/.config:/root/.config \
          test-provisioning:$GITHUB_SHA
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID }}
    - name: Run Monitoring Integration Tests
      timeout-minutes: 30
      run: |
        set -x
        # install dependencies
        python3 -m pip install -r tests/requirements.txt
        # authenticate cluster
        CLUSTER_ZONE=$(gcloud container clusters list --filter="name:cloud-ops-sandbox" --project $PROJECT_ID --format="value(zone)")
        gcloud container clusters get-credentials cloud-ops-sandbox --zone "$CLUSTER_ZONE"
        # run tests
        python3 tests/monitoring_integration_test.py $PROJECT_ID
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Clean Project State
      if: ${{ always()  }}
      timeout-minutes: 30
      run: |
        # clean GCP resources
        .github/workflows/e2e_scripts/clean.sh
<commit_msg>fix image path<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: "End to End - Latest"
on:
  push:
    # run on pushes to master or release/*
    branches:
      - master
      - release/*
      - e2e*
      - loadgen_integration # will remove before merging
  workflow_dispatch:
    # trigger through UI or API
jobs:
  end-to-end-latest:
    runs-on: [self-hosted, e2e-worker]
    steps:
    - uses: actions/checkout@v2
    - name: Clean Project State
      timeout-minutes: 30
      run: |
        # clean GCP resources
        .github/workflows/e2e_scripts/clean.sh
    - name: Build Sandbox Services
      run: |
        set -x
        skaffold config set --global local-cluster false
        # tag with git hash
        skaffold build --default-repo=gcr.io/$PROJECT_ID/tests \
                       --tag=$GITHUB_SHA
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Build Load Generator
      run: |
        set -x
        gcloud auth configure-docker --quiet
        # push latest load generator to test image repository
        IMAGE_PATH=gcr.io/$PROJECT_ID/tests/gcr.io/stackdriver-sandbox-230822/sandbox/loadgenerator/basic:$GITHUB_SHA
        docker build -t $IMAGE_PATH ./src/loadgenerator
        docker push $IMAGE_PATH
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Rewrite Manifests
      run: |
        set -x
        OLD_PATH=gcr.io/stackdriver-sandbox-230822
        NEW_PATH=gcr.io/stackdriver-sandbox-e2e/tests/$OLD_PATH
        # use newly built containers instead of latest pushed
        find "./kubernetes-manifests" -name '*.yaml' -exec sed -i -e "s#$OLD_PATH#$NEW_PATH#g" {} \;
        find "./kubernetes-manifests" -name '*.yaml' -exec sed -i -e "s#:latest#:$GITHUB_SHA#g" {} \;
    - name: Run Install Script
      timeout-minutes: 30
      run: |
        set -x
        # build cloud shell image
        docker build -t test-cloud-shell:$GITHUB_SHA ./cloud-shell
        # run install script
        docker run --rm \
          -e project_id=$PROJECT_ID \
          -e skip_workspace_prompt=1 \
          -e service_wait=1 \
          -v ~/.config:/root/.config \
          -v `pwd`:/sandbox-shared \
          --entrypoint /sandbox-shared/.github/workflows/e2e_scripts/run_install.sh \
          test-cloud-shell:$GITHUB_SHA
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Test Using Existing Project
      timeout-minutes: 30
      run: |
        set -x
        # run install script
        docker run --rm \
          -e project_id=$PROJECT_ID \
          -e skip_workspace_prompt=1 \
          -e service_wait=1 \
          -v ~/.config:/root/.config \
          -v `pwd`:/sandbox-shared \
          --entrypoint /sandbox-shared/.github/workflows/e2e_scripts/run_install.sh \
          test-cloud-shell:$GITHUB_SHA
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Run Provisioning Test
      timeout-minutes: 30
      run: |
        set -x
        # get cluster zones
        CLUSTER_ZONE=$(gcloud container clusters list --filter="name:cloud-ops-sandbox" --project $PROJECT_ID --format="value(zone)")
        LOADGEN_ZONE=$(gcloud container clusters list --filter="name:loadgenerator" --project $PROJECT_ID --format="value(zone)")
        # build provisioning test image
        docker build -t test-provisioning:$GITHUB_SHA tests/provisioning/.
        # run provisioning tests
        docker run --rm \
          -e GOOGLE_CLOUD_PROJECT=$PROJECT_ID \
          -e ZONE=$CLUSTER_ZONE \
          -e LOADGEN_ZONE=$LOADGEN_ZONE \
          -v ~/.config:/root/.config \
          test-provisioning:$GITHUB_SHA
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID }}
    - name: Run Monitoring Integration Tests
      timeout-minutes: 30
      run: |
        set -x
        # install dependencies
        python3 -m pip install -r tests/requirements.txt
        # authenticate cluster
        CLUSTER_ZONE=$(gcloud container clusters list --filter="name:cloud-ops-sandbox" --project $PROJECT_ID --format="value(zone)")
        gcloud container clusters get-credentials cloud-ops-sandbox --zone "$CLUSTER_ZONE"
        # run tests
        python3 tests/monitoring_integration_test.py $PROJECT_ID
      env:
        PROJECT_ID: ${{ secrets.E2E_PROJECT_ID  }}
    - name: Clean Project State
      if: ${{ always()  }}
      timeout-minutes: 30
      run: |
        # clean GCP resources
        .github/workflows/e2e_scripts/clean.sh

{'fraction_non_alphanumeric': 0.1042186097536651, 'fraction_numerical': 0.01037199561184801, 'mean_word_length': 2.149497487437186, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1533581', 'n_tokens_mistral': 3245, 'n_tokens_neox': 3070, 'n_words': 808}