<commit_before>steps:

  # Linux Specific
  - script: |
      sudo apt-get update
      sudo apt-get install -y wget software-properties-common
      sudo apt-get install -y build-essential ca-certificates xvfb fakeroot git libsecret-1-dev libx11-dev libxkbfile-dev xz-utils xorriso zsync libxss1 libgtk-3-0 libasound2 libicu-dev software-properties-common
      # rpm needs to be at least 4.14, in xenial it is too old
      sudo apt-add-repository -y 'deb https://azure.archive.ubuntu.com/ubuntu/ bionic main universe'
      sudo apt update
      sudo apt-get install -y rpm
      # clang 9 is included in the image
      clang -v
    displayName: Install apt dependencies
    condition: eq(variables['Agent.OS'], 'Linux')


  - script: sudo /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16
    displayName: Start Xvfb
    condition: eq(variables['Agent.OS'], 'Linux')

  # Common
  - pwsh: |
      if ($env:BUILD_ARCH -eq "x86") {
        echo "##vso[task.setvariable variable=IsWinX86]true"
      }
    displayName: Set "IsWinX86" based on "BUILD_ARCH"
    # Convert "BUILD_ARCH" to a boolean ("IsWinX86") for the following NodeTool task.
    # Templates evaluate too early for the matrix variable "BUILD_ARCH" to be available in ${{ template expressions }}.
    # Scripts are interpreted at runtime, so "BUILD_ARCH" will be available to this script, and we can set "IsWinX86".
<commit_msg>Azure only has http and no https<commit_after>steps:

  # Linux Specific
  - script: |
      sudo apt-get update
      sudo apt-get install -y wget software-properties-common
      sudo apt-get install -y build-essential ca-certificates xvfb fakeroot git libsecret-1-dev libx11-dev libxkbfile-dev xz-utils xorriso zsync libxss1 libgtk-3-0 libasound2 libicu-dev software-properties-common
      # rpm needs to be at least 4.14, in xenial it is too old
      sudo apt-add-repository -y 'deb http://azure.archive.ubuntu.com/ubuntu/ bionic main universe'
      sudo apt update
      sudo apt-get install -y rpm
      # clang 9 is included in the image
      clang -v
    displayName: Install apt dependencies
    condition: eq(variables['Agent.OS'], 'Linux')


  - script: sudo /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16
    displayName: Start Xvfb
    condition: eq(variables['Agent.OS'], 'Linux')

  # Common
  - pwsh: |
      if ($env:BUILD_ARCH -eq "x86") {
        echo "##vso[task.setvariable variable=IsWinX86]true"
      }
    displayName: Set "IsWinX86" based on "BUILD_ARCH"
    # Convert "BUILD_ARCH" to a boolean ("IsWinX86") for the following NodeTool task.
    # Templates evaluate too early for the matrix variable "BUILD_ARCH" to be available in ${{ template expressions }}.
    # Scripts are interpreted at runtime, so "BUILD_ARCH" will be available to this script, and we can set "IsWinX86".

{'fraction_non_alphanumeric': 0.10059563203176704, 'fraction_numerical': 0.02382528127068167, 'mean_word_length': 3.0906630581867387, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5993209', 'n_tokens_mistral': 1033, 'n_tokens_neox': 942, 'n_words': 324}