<commit_before>steps:
  - task: NodeTool@0
    inputs:
      versionSpec: "16.x"

  - task: AzureKeyVault@1
    displayName: "Azure Key Vault: Get Secrets"
    inputs:
      azureSubscription: "vscode-builds-subscription"
      KeyVaultName: vscode
      SecretsFilter: "github-distro-mixin-password,ESRP-PKI,esrp-aad-username,esrp-aad-password"

  - task: DownloadPipelineArtifact@2
    inputs:
      artifact: Compilation
      path: $(Build.ArtifactStagingDirectory)
    displayName: Download compilation output

  - task: DownloadPipelineArtifact@2
    inputs:
      artifact: reh_node_modules-$(VSCODE_ARCH)
      path: $(Build.ArtifactStagingDirectory)
    displayName: Download server build dependencies
    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'))

  - script: |
      set -e
      tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz
    displayName: Extract compilation output

  - script: |
      set -e
      cat << EOF > ~/.netrc
      machine github.com
      login vscode
      password $(<PASSWORD>-distro-mixin-password)
      EOF

      git config user.email "<EMAIL>"
      git config user.name "VSCode"
    displayName: Prepare tooling

  - script: |
      set -e
      git pull --no-rebase https://github.com/$(VSCODE_MIXIN_REPO).git $(node -p "require('./package.json').distro")
    displayName: Merge distro

  - script: |
      mkdir -p .build
      node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash
    displayName: Prepare yarn cache flags

  - task: Cache@2
    inputs:
      key: "nodeModules | $(Agent.OS) | .build/yarnlockhash"
      path: .build/node_modules_cache
      cacheHitVar: NODE_MODULES_RESTORED
    displayName: Restore node_modules cache

  - script: |
      set -e
      tar -xzf .build/node_modules_cache/cache.tgz
    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
    displayName: Extract node_modules cache

  - script: |
      set -e
      npx https://aka.ms/enablesecurefeed standAlone
    timeoutInMinutes: 5
    retryCountOnTaskFailure: 3
    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), eq(variables['ENABLE_TERRAPIN'], 'true'))
    displayName: Switch to Terrapin packages

  - script: |
      set -e
      for i in {1..3}; do # try 3 times, for Terrapin
        yarn --cwd build --frozen-lockfile && break
        if [ $i -eq 3 ]; then
          echo "Yarn failed too many times" >&2
          exit 1
        fi
        echo "Yarn failed $i, trying again..."
      done
    displayName: Install build dependencies

  - script: |
      set -e
      export npm_config_arch=$(NPM_ARCH)

      # Download clang based on chromium revision used by vscode
      curl -s https://raw.githubusercontent.com/chromium/chromium/96.0.4664.110/tools/clang/scripts/update.py | python - --output-dir=$PWD/.build/CR_Clang --host-os=linux
      # Download libcxx headers and objects from upstream electron releases
      DEBUG=libcxx-fetcher \
      VSCODE_LIBCXX_OBJECTS_DIR=$PWD/.build/libcxx-objects \
      VSCODE_LIBCXX_HEADERS_DIR=$PWD/.build/libcxx_headers  \
      VSCODE_LIBCXXABI_HEADERS_DIR=$PWD/.build/libcxxabi_headers \
      VSCODE_ARCH="$(NPM_ARCH)" \
      node build/linux/libcxx-fetcher.js
      # Set compiler toolchain
      # Flags for the client build are based on
      # https://source.chromium.org/chromium/chromium/src/+/refs/tags/96.0.4664.110:build/config/arm.gni
      # https://source.chromium.org/chromium/chromium/src/+/refs/tags/96.0.4664.110:build/config/compiler/BUILD.gn
      # https://source.chromium.org/chromium/chromium/src/+/refs/tags/96.0.4664.110:build/config/c++/BUILD.gn
      export CC=$PWD/.build/CR_Clang/bin/clang
      export CXX=$PWD/.build/CR_Clang/bin/clang++
      export CXXFLAGS="-nostdinc++ -D__NO_INLINE__ -isystem$PWD/.build/libcxx_headers -isystem$PWD/.build/libcxx_headers/include -isystem$PWD/.build/libcxxabi_headers/include -fPIC -flto=thin -fsplit-lto-unit"
      export CFLAGS="-nostdinc++ -D__NO_INLINE__ -isystem$PWD/.build/libcxx_headers -isystem$PWD/.build/libcxx_headers/include -isystem$PWD/.build/libcxxabi_headers/include -fPIC -flto=thin -fsplit-lto-unit"
      export LDFLAGS="-stdlib=libc++ -fuse-ld=lld -flto=thin -fsplit-lto-unit -L$PWD/.build/libcxx-objects -lc++abi"

      if [ "$VSCODE_ARCH" == "arm64" ]; then
        export CXXFLAGS="${CXXFLAGS} --target=aarch64-linux-gnu -fwhole-program-vtables"
        export CFLAGS="${CFLAGS} --target=aarch64-linux-gnu -fwhole-program-vtables"
        export LDFLAGS="${LDFLAGS} --target=aarch64-linux-gnu -march=arm64 -fwhole-program-vtables"
        export VSCODE_REMOTE_CC=/usr/bin/aarch64-linux-gnu-gcc
        export VSCODE_REMOTE_CXX=/usr/bin/aarch64-linux-gnu-g++
      fi

      if [ "$VSCODE_ARCH" == "armhf" ]; then
        export CXXFLAGS="${CXXFLAGS} --target=arm-linux-gnueabihf -march=armv7-a -mthumb -mfloat-abi=hard -mfpu=neon -mtune=generic-armv7-a"
        export CFLAGS="${CFLAGS} --target=arm-linux-gnueabihf -march=armv7-a -mthumb -mfloat-abi=hard -mfpu=neon -mtune=generic-armv7-a"
        export LDFLAGS="${LDFLAGS} --target=arm-linux-gnueabihf -march=armv7-a"
        export VSCODE_REMOTE_CC=/usr/bin/arm-linux-gnueabihf-gcc
        export VSCODE_REMOTE_CXX=/usr/bin/arm-linux-gnueabihf-g++
      fi

      for i in {1..3}; do # try 3 times, for Terrapin
        yarn --frozen-lockfile && break
        if [ $i -eq 3 ]; then
          echo "Yarn failed too many times" >&2
          exit 1
        fi
        echo "Yarn failed $i, trying again..."
      done
    env:
      ELECTRON_SKIP_BINARY_DOWNLOAD: 1
      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
      GITHUB_TOKEN: "$(github-distro-mixin-password)"
    displayName: Install dependencies<commit_msg>ci: fix default gcc choice for remote (#142554)<commit_after>steps:
  - task: NodeTool@0
    inputs:
      versionSpec: "16.x"

  - task: AzureKeyVault@1
    displayName: "Azure Key Vault: Get Secrets"
    inputs:
      azureSubscription: "vscode-builds-subscription"
      KeyVaultName: vscode
      SecretsFilter: "github-distro-mixin-password,ESRP-PKI,esrp-aad-username,esrp-aad-password"

  - task: DownloadPipelineArtifact@2
    inputs:
      artifact: Compilation
      path: $(Build.ArtifactStagingDirectory)
    displayName: Download compilation output

  - task: DownloadPipelineArtifact@2
    inputs:
      artifact: reh_node_modules-$(VSCODE_ARCH)
      path: $(Build.ArtifactStagingDirectory)
    displayName: Download server build dependencies
    condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'))

  - script: |
      set -e
      tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz
    displayName: Extract compilation output

  - script: |
      set -e
      cat << EOF > ~/.netrc
      machine github.com
      login vscode
      password $(github-distro-mixin-password)
      EOF

      git config user.email "<EMAIL>"
      git config user.name "VSCode"
    displayName: Prepare tooling

  - script: |
      set -e
      git pull --no-rebase https://github.com/$(VSCODE_MIXIN_REPO).git $(node -p "require('./package.json').distro")
    displayName: Merge distro

  - script: |
      mkdir -p .build
      node build/azure-pipelines/common/computeNodeModulesCacheKey.js $VSCODE_ARCH $ENABLE_TERRAPIN > .build/yarnlockhash
    displayName: Prepare yarn cache flags

  - task: Cache@2
    inputs:
      key: "nodeModules | $(Agent.OS) | .build/yarnlockhash"
      path: .build/node_modules_cache
      cacheHitVar: NODE_MODULES_RESTORED
    displayName: Restore node_modules cache

  - script: |
      set -e
      tar -xzf .build/node_modules_cache/cache.tgz
    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
    displayName: Extract node_modules cache

  - script: |
      set -e
      npx https://aka.ms/enablesecurefeed standAlone
    timeoutInMinutes: 5
    retryCountOnTaskFailure: 3
    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), eq(variables['ENABLE_TERRAPIN'], 'true'))
    displayName: Switch to Terrapin packages

  - script: |
      set -e
      for i in {1..3}; do # try 3 times, for Terrapin
        yarn --cwd build --frozen-lockfile && break
        if [ $i -eq 3 ]; then
          echo "Yarn failed too many times" >&2
          exit 1
        fi
        echo "Yarn failed $i, trying again..."
      done
    displayName: Install build dependencies

  - script: |
      set -e
      export npm_config_arch=$(NPM_ARCH)

      # Download clang based on chromium revision used by vscode
      curl -s https://raw.githubusercontent.com/chromium/chromium/96.0.4664.110/tools/clang/scripts/update.py | python - --output-dir=$PWD/.build/CR_Clang --host-os=linux
      # Download libcxx headers and objects from upstream electron releases
      DEBUG=libcxx-fetcher \
      VSCODE_LIBCXX_OBJECTS_DIR=$PWD/.build/libcxx-objects \
      VSCODE_LIBCXX_HEADERS_DIR=$PWD/.build/libcxx_headers  \
      VSCODE_LIBCXXABI_HEADERS_DIR=$PWD/.build/libcxxabi_headers \
      VSCODE_ARCH="$(NPM_ARCH)" \
      node build/linux/libcxx-fetcher.js
      # Set compiler toolchain
      # Flags for the client build are based on
      # https://source.chromium.org/chromium/chromium/src/+/refs/tags/96.0.4664.110:build/config/arm.gni
      # https://source.chromium.org/chromium/chromium/src/+/refs/tags/96.0.4664.110:build/config/compiler/BUILD.gn
      # https://source.chromium.org/chromium/chromium/src/+/refs/tags/96.0.4664.110:build/config/c++/BUILD.gn
      export CC=$PWD/.build/CR_Clang/bin/clang
      export CXX=$PWD/.build/CR_Clang/bin/clang++
      export CXXFLAGS="-nostdinc++ -D__NO_INLINE__ -isystem$PWD/.build/libcxx_headers -isystem$PWD/.build/libcxx_headers/include -isystem$PWD/.build/libcxxabi_headers/include -fPIC -flto=thin -fsplit-lto-unit"
      export CFLAGS="-nostdinc++ -D__NO_INLINE__ -isystem$PWD/.build/libcxx_headers -isystem$PWD/.build/libcxx_headers/include -isystem$PWD/.build/libcxxabi_headers/include -fPIC -flto=thin -fsplit-lto-unit"
      export LDFLAGS="-stdlib=libc++ -fuse-ld=lld -flto=thin -fsplit-lto-unit -L$PWD/.build/libcxx-objects -lc++abi"
      export VSCODE_REMOTE_CC=$(which gcc)
      export VSCODE_REMOTE_CXX=$(which g++)

      if [ "$VSCODE_ARCH" == "arm64" ]; then
        export CXXFLAGS="${CXXFLAGS} --target=aarch64-linux-gnu -fwhole-program-vtables"
        export CFLAGS="${CFLAGS} --target=aarch64-linux-gnu -fwhole-program-vtables"
        export LDFLAGS="${LDFLAGS} --target=aarch64-linux-gnu -march=arm64 -fwhole-program-vtables"
        export VSCODE_REMOTE_CC=/usr/bin/aarch64-linux-gnu-gcc
        export VSCODE_REMOTE_CXX=/usr/bin/aarch64-linux-gnu-g++
      fi

      if [ "$VSCODE_ARCH" == "armhf" ]; then
        export CXXFLAGS="${CXXFLAGS} --target=arm-linux-gnueabihf -march=armv7-a -mthumb -mfloat-abi=hard -mfpu=neon -mtune=generic-armv7-a"
        export CFLAGS="${CFLAGS} --target=arm-linux-gnueabihf -march=armv7-a -mthumb -mfloat-abi=hard -mfpu=neon -mtune=generic-armv7-a"
        export LDFLAGS="${LDFLAGS} --target=arm-linux-gnueabihf -march=armv7-a"
        export VSCODE_REMOTE_CC=/usr/bin/arm-linux-gnueabihf-gcc
        export VSCODE_REMOTE_CXX=/usr/bin/arm-linux-gnueabihf-g++
      fi

      for i in {1..3}; do # try 3 times, for Terrapin
        yarn --frozen-lockfile && break
        if [ $i -eq 3 ]; then
          echo "Yarn failed too many times" >&2
          exit 1
        fi
        echo "Yarn failed $i, trying again..."
      done
    env:
      ELECTRON_SKIP_BINARY_DOWNLOAD: 1
      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
      GITHUB_TOKEN: <PASSWORD>)"
    displayName: Install dependencies
{'fraction_non_alphanumeric': 0.1243363589655763, 'fraction_numerical': 0.014899811611577324, 'mean_word_length': 2.9336476928258675, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '528962', 'n_tokens_mistral': 4304, 'n_tokens_neox': 4057, 'n_words': 768}