<commit_before>#
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
steps:

  #
  # Extract the cache
  #
  # The gradle build cache is stored as a tarball in Google Cloud Storage to
  # make builds faster.
  #
  # After extracting the cache to the /build_cache directory, we need to supply
  # that to gradle, and include the volume in steps that require the cache.
  #
  - name: 'gcr.io/cloud-builders/gsutil'
    id: copy_build_cache
    # we use rsync and not cp so that this step doesn't fail the first time it's run
    args: ['rsync', 'gs://${_CACHE_BUCKET}/', '/build_cache']
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

  - name: 'gcr.io/$PROJECT_ID/tar'
    id: extract_build_cache
    waitFor: ['copy_build_cache']
    # This might fail the first time, but that's okay
    entrypoint: 'bash'
    args:
      - '-c'
      - |
        tar xpzf /build_cache/cache.tgz -C /build_cache || echo "No cache found."
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

  # TODO: Remove this step when .git can be explicitly included in the tarball using .gcloudignore
  # https://github.com/GoogleCloudPlatform/cloud-builders/issues/401
  - name: 'gcr.io/cloud-builders/git'
    id: copy_git_directory
    waitFor: ['extract_build_cache']
    entrypoint: 'bash'
    args:
      - '-c'
      - |
        git clone --branch master $_HEAD_REPO_URL tmp \
        && mv tmp/.git . \
        && rm -rf tmp

  #
  # Build the project
  #
  - name: 'gcr.io/$PROJECT_ID/android:28'
    id: build
    args: ["./gradlew", "assembleDebug"]
    env:
      - 'TERM=dumb'
      - 'JAVA_TOOL_OPTIONS="-Xmx3g"'
      - 'GRADLE_USER_HOME=/build_cache/.gradle'
      - 'GRADLE_OPTS="-Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=8 -Dkotlin.incremental=false"'
      - 'BRANCH_NAME=$BRANCH_NAME'
    waitFor: ['copy_git_directory']
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

  #
  # Save the APKs
  #
  - name: 'gcr.io/cloud-builders/gsutil'
    args: ['-q', 'cp', '-r', 'gnd/build/outputs/apk', 'gs://${_ARTIFACT_BUCKET}/$BRANCH_NAME-$BUILD_ID/']
    waitFor: ['build']

  #
  # Cleanup
  #

  # Compress the gradle build cache
  - name: 'gcr.io/$PROJECT_ID/tar'
    id: compress_cache
    args: ['cpvzf', '/build_cache/cache.tgz', '-C', '/build_cache', '/build_cache/.gradle']
    waitFor: ['build']
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

  # Store the build cache
  - name: gcr.io/cloud-builders/gsutil
    args: ['cp', '/build_cache/cache.tgz', 'gs://${_CACHE_BUCKET}/cache.tgz']
    waitFor: ['compress_cache']
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

timeout: 1800s<commit_msg>Use base docker image for compressing gradle cache<commit_after>#
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Steps for cloud build:
#
# -----------------------------------------------------------------------------------------------------------------------------------
# Copy gradle cache ==> Extract tarball ==> Copy .git dir ==> Build debug apk ==> Save apk ==> Compress gradle files ==> Save tarball
# -----------------------------------------------------------------------------------------------------------------------------------
#
# Based on https://github.com/GoogleCloudPlatform/cloud-builders-community
steps:

  #
  # Extract the cache
  #
  # The gradle build cache is stored as a tarball in Google Cloud Storage to
  # make builds faster.
  #
  # After extracting the cache to the /build_cache directory, we need to supply
  # that to gradle, and include the volume in steps that require the cache.
  #
  - name: 'gcr.io/cloud-builders/gsutil'
    id: copy_build_cache
    # we use rsync and not cp so that this step doesn't fail the first time it's run
    args: ['rsync', 'gs://${_CACHE_BUCKET}/', '/build_cache']
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

  - name: 'gcr.io/$PROJECT_ID/android:base'
    id: extract_build_cache
    waitFor: ['copy_build_cache']
    # This might fail the first time, but that's okay
    entrypoint: 'bash'
    args:
      - '-c'
      - |
        tar xpzf /build_cache/cache.tgz -C /build_cache || echo "No cache found."
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

  # TODO: Remove this step when .git can be explicitly included in the tarball using .gcloudignore
  # https://github.com/GoogleCloudPlatform/cloud-builders/issues/401
  - name: 'gcr.io/cloud-builders/git'
    id: copy_git_directory
    waitFor: ['extract_build_cache']
    entrypoint: 'bash'
    args:
      - '-c'
      - |
        git clone --branch master $_HEAD_REPO_URL tmp \
        && mv tmp/.git . \
        && rm -rf tmp

  #
  # Build the project
  #
  - name: 'gcr.io/$PROJECT_ID/android:28'
    id: build
    args: ["./gradlew", "assembleDebug"]
    env:
      - 'TERM=dumb'
      - 'JAVA_TOOL_OPTIONS="-Xmx3g"'
      - 'GRADLE_USER_HOME=/build_cache/.gradle'
      - 'GRADLE_OPTS="-Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=8 -Dkotlin.incremental=false"'
      - 'BRANCH_NAME=$BRANCH_NAME'
    waitFor: ['copy_git_directory']
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

  #
  # Save the APKs
  #
  - name: 'gcr.io/cloud-builders/gsutil'
    args: ['-q', 'cp', '-r', 'gnd/build/outputs/apk', 'gs://${_ARTIFACT_BUCKET}/$BRANCH_NAME-$BUILD_ID/']
    waitFor: ['build']

  #
  # Cleanup
  #

  # Compress the gradle build cache
  - name: 'gcr.io/$PROJECT_ID/android:base'
    id: compress_cache
    waitFor: ['build']
    entrypoint: 'bash'
    args:
      - '-c'
      - |
        tar cpvzf /build_cache/cache.tgz -C /build_cache /build_cache/.gradle
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

  # Store the build cache
  - name: gcr.io/cloud-builders/gsutil
    args: ['cp', '/build_cache/cache.tgz', 'gs://${_CACHE_BUCKET}/cache.tgz']
    waitFor: ['compress_cache']
    volumes:
      - name: 'build_cache'
        path: '/build_cache'

timeout: 1800s
{'fraction_non_alphanumeric': 0.15959539196403486, 'fraction_numerical': 0.005338578252318067, 'mean_word_length': 2.4541484716157207, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 5, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '178450', 'n_tokens_mistral': 2380, 'n_tokens_neox': 2302, 'n_words': 655}