<commit_before># Copyright 2021 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

# Pipeline steps for https://buildkite.com/iree/iree-benchmark

steps:
  - label: "Build"
    commands:
      - "docker run --user=$(id -u):$(id -g) --volume=\\${HOME?}:\\${HOME?} --volume=/etc/passwd:/etc/passwd:ro --volume=/etc/group:/etc/group:ro --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/frontends@sha256:58adb4131cfc7b08cd5767c577420f3479ca2f46bb67d9fac6c0797984627758 build_tools/cmake/build_android_benchmark.sh"
      - "tar --exclude='*.tar.gz' --exclude='*.tgz' --exclude='*.mlir' --exclude='*.tflite' -czvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz build-host/benchmark_suites"
      - "tar -czvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz build-android/iree/tools/iree-benchmark-module build-android-trace/iree/tools/iree-benchmark-module"
    if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark')"
    agents:
      - "queue=build"
    env:
      IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
    artifact_paths:
      - "benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz"

  - wait

  - label: "Benchmark on Pixel 4 (snapdragon-855, adreno-640)"
    commands:
      - "git clean -f"
      - "buildkite-agent artifact download --step Build benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz ./"
      - "buildkite-agent artifact download --step Build iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz ./"
      - "wget https://storage.googleapis.com/iree-shared-files/tracy-capture-058e8901.tgz"
      - "tar -xzvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "tar -xzvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "tar -xzvf tracy-capture-058e8901.tgz"
      - "python3 build_tools/benchmarks/run_benchmarks_on_android.py --normal_benchmark_tool=build-android/iree/tools/iree-benchmark-module --traced_benchmark_tool=build-android-trace/iree/tools/iree-benchmark-module --trace_capture_tool=tracy-capture -o benchmark-results-pixel-4-${BUILDKITE_BUILD_NUMBER}.json --capture_tarball=trace-captures-pixel-4-${BUILDKITE_BUILD_NUMBER}.tgz --verbose build-host/"
    if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark')"
    agents:
      - "android-soc=snapdragon-855"
      - "android-version=11"
      - "queue=benchmark-android"
    artifact_paths:
      - "benchmark-results-pixel-4-${BUILDKITE_BUILD_NUMBER}.json"
      - "trace-captures-pixel-4-${BUILDKITE_BUILD_NUMBER}.tgz"
    timeout_in_minutes: "60"

  - label: "Benchmark on Galaxy S20 (exynos-990, mali-g77)"
    commands:
      - "git clean -f"
      - "buildkite-agent artifact download --step Build benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz ./"
      - "buildkite-agent artifact download --step Build iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz ./"
      - "wget https://storage.googleapis.com/iree-shared-files/tracy-capture-058e8901.tgz"
      - "tar -xzvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "tar -xzvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "tar -xzvf tracy-capture-058e8901.tgz"
      - "python3 build_tools/benchmarks/run_benchmarks_on_android.py --normal_benchmark_tool=build-android/iree/tools/iree-benchmark-module --traced_benchmark_tool=build-android-trace/iree/tools/iree-benchmark-module --trace_capture_tool=tracy-capture -o benchmark-results-galaxy-s20-${BUILDKITE_BUILD_NUMBER}.json --capture_tarball=trace-captures-galaxy-s20-${BUILDKITE_BUILD_NUMBER}.tgz --verbose build-host/"
    if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark')"
    agents:
      - "android-soc=exynos-990"
      - "android-version=11"
      - "queue=benchmark-android"
    artifact_paths:
      - "benchmark-results-galaxy-s20-${BUILDKITE_BUILD_NUMBER}.json"
      - "trace-captures-galaxy-s20-${BUILDKITE_BUILD_NUMBER}.tgz"
    timeout_in_minutes: "60"

  - wait

  - label: "Comment benchmark results on pull request"
    commands:
      - "git clean -f"
      - "buildkite-agent artifact download benchmark-results-*.json ./"
      - "python3 build_tools/benchmarks/post_benchmarks_as_pr_comment.py --verbose --query-base benchmark-results-*.json"
    key: "post-on-pr"
    if: "build.pull_request.id != null && (build.pull_request.labels includes 'buildkite:benchmark')"
    agents:
      - "queue=report"

  - label: "Push benchmark results to dashboard"
    commands:
      - "git clean -f"
      - "buildkite-agent artifact download benchmark-results-*.json ./"
      - "python3 build_tools/benchmarks/upload_benchmarks_to_dashboard.py --verbose benchmark-results-*.json"
    key: "upload-to-dashboard"<commit_msg>Perform deep clean for benchmark steps (#7722)<commit_after># Copyright 2021 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

# Pipeline steps for https://buildkite.com/iree/iree-benchmark

steps:
  - label: "Build"
    commands:
      - "docker run --user=$(id -u):$(id -g) --volume=\\${HOME?}:\\${HOME?} --volume=/etc/passwd:/etc/passwd:ro --volume=/etc/group:/etc/group:ro --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/frontends@sha256:58adb4131cfc7b08cd5767c577420f3479ca2f46bb67d9fac6c0797984627758 build_tools/cmake/build_android_benchmark.sh"
      - "tar --exclude='*.tar.gz' --exclude='*.tgz' --exclude='*.mlir' --exclude='*.tflite' -czvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz build-host/benchmark_suites"
      - "tar -czvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz build-android/iree/tools/iree-benchmark-module build-android-trace/iree/tools/iree-benchmark-module"
    if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark')"
    agents:
      - "queue=build"
    env:
      IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
    artifact_paths:
      - "benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz"

  - wait

  - label: "Benchmark on Pixel 4 (snapdragon-855, adreno-640)"
    commands:
      - "git clean -fdx"
      - "buildkite-agent artifact download --step Build benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz ./"
      - "buildkite-agent artifact download --step Build iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz ./"
      - "wget https://storage.googleapis.com/iree-shared-files/tracy-capture-058e8901.tgz"
      - "tar -xzvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "tar -xzvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "tar -xzvf tracy-capture-058e8901.tgz"
      - "python3 build_tools/benchmarks/run_benchmarks_on_android.py --normal_benchmark_tool=build-android/iree/tools/iree-benchmark-module --traced_benchmark_tool=build-android-trace/iree/tools/iree-benchmark-module --trace_capture_tool=tracy-capture -o benchmark-results-pixel-4-${BUILDKITE_BUILD_NUMBER}.json --capture_tarball=trace-captures-pixel-4-${BUILDKITE_BUILD_NUMBER}.tgz --verbose build-host/"
    if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark')"
    agents:
      - "android-soc=snapdragon-855"
      - "android-version=11"
      - "queue=benchmark-android"
    artifact_paths:
      - "benchmark-results-pixel-4-${BUILDKITE_BUILD_NUMBER}.json"
      - "trace-captures-pixel-4-${BUILDKITE_BUILD_NUMBER}.tgz"
    timeout_in_minutes: "60"

  - label: "Benchmark on Galaxy S20 (exynos-990, mali-g77)"
    commands:
      - "git clean -fdx"
      - "buildkite-agent artifact download --step Build benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz ./"
      - "buildkite-agent artifact download --step Build iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz ./"
      - "wget https://storage.googleapis.com/iree-shared-files/tracy-capture-058e8901.tgz"
      - "tar -xzvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "tar -xzvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz"
      - "tar -xzvf tracy-capture-058e8901.tgz"
      - "python3 build_tools/benchmarks/run_benchmarks_on_android.py --normal_benchmark_tool=build-android/iree/tools/iree-benchmark-module --traced_benchmark_tool=build-android-trace/iree/tools/iree-benchmark-module --trace_capture_tool=tracy-capture -o benchmark-results-galaxy-s20-${BUILDKITE_BUILD_NUMBER}.json --capture_tarball=trace-captures-galaxy-s20-${BUILDKITE_BUILD_NUMBER}.tgz --verbose build-host/"
    if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark')"
    agents:
      - "android-soc=exynos-990"
      - "android-version=11"
      - "queue=benchmark-android"
    artifact_paths:
      - "benchmark-results-galaxy-s20-${BUILDKITE_BUILD_NUMBER}.json"
      - "trace-captures-galaxy-s20-${BUILDKITE_BUILD_NUMBER}.tgz"
    timeout_in_minutes: "60"

  - wait

  - label: "Comment benchmark results on pull request"
    commands:
      - "git clean -fdx"
      - "buildkite-agent artifact download benchmark-results-*.json ./"
      - "python3 build_tools/benchmarks/post_benchmarks_as_pr_comment.py --verbose --query-base benchmark-results-*.json"
    key: "post-on-pr"
    if: "build.pull_request.id != null && (build.pull_request.labels includes 'buildkite:benchmark')"
    agents:
      - "queue=report"

  - label: "Push benchmark results to dashboard"
    commands:
      - "git clean -fdx"
      - "buildkite-agent artifact download benchmark-results-*.json ./"
      - "python3 build_tools/benchmarks/upload_benchmarks_to_dashboard.py --verbose benchmark-results-*.json"
    key: "upload-to-dashboard"
{'fraction_non_alphanumeric': 0.14565064005644593, 'fraction_numerical': 0.025803850418304606, 'mean_word_length': 3.6868209730751063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3313484', 'n_tokens_mistral': 3723, 'n_tokens_neox': 3439, 'n_words': 489}