<commit_before>name: ci

on:
  push:
    branches:
      # only on merges to master branch
      - master
      # and version branches, which only include minor versions (eg: v3.4)
      - v[0-9]+.[0-9]+
    tags:
      # also version tags, which include bugfix releases (eg: v3.4.0)
      - v[0-9]+.[0-9]+.[0-9]+
  pull_request:
    type: [opened, reopened, edited]
    branches:
      # Only for PRs targeting those branches
      - master
      - v[0-9]+.[0-9]+
  schedule:
    # run every night at midnight
    - cron:  '0 0 * * *'

jobs:
  ci:
    name: '${{ matrix.name }} - python (${{ matrix.python-version }})'
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: 'Lint Checks'
            task: 'ci-checks'
            python-version: '3.6'
          - name: 'Compile'
            task: 'ci-compile'
            python-version: '3.6'
          - name: 'Pack Tests'
            task: 'ci-packs-tests'
            python-version: '3.6'
          - name: 'Unit Tests'
            task: 'ci-unit'
            python-version: '3.6'
          # This job is slow so we only run in on a daily basis
          # - name: 'Micro Benchmarks'
          #   task: 'micro-benchmarks'
          #   python-version: '3.6'
          - name: 'Integration Tests'
            task: 'ci-integration'
            python-version: '3.6'
    services:
      mongo:
        image: mongo:4.0
        ports:
          - 27017:27017

      # In GHA, these services are started first before the code is checked out.
      # We use bitnami images to facilitate reconfiguring RabbitMQ during ci-integration tests.
      # We rely on custom config and SSL certs that are in the repo.
      # Many images require config in env vars (which we can't change during the test job)
      # or they require config in entrypoint args (which we can't override for GHA services)
      # bitnami builds ways to get config files from mounted volumes.
      rabbitmq:
        image: bitnami/rabbitmq:3.8
        volumes:
          - /home/runner/rabbitmq_conf:/bitnami/conf  # RABBITMQ_MOUNTED_CONF_DIR
        env:
          # tell bitnami/rabbitmq to enable this by default
          RABBITMQ_PLUGINS: rabbitmq_management
          RABBITMQ_USERNAME: guest
          RABBITMQ_PASSWORD: <PASSWORD>

        # These are strictly docker options, not entrypoint args (GHA restriction)
        options: >-
          --name rabbitmq

        ports:
          # These 6 ports are exposed by bitnami/rabbitmq (see https://www.rabbitmq.com/networking.html#ports)
          # host_port:container_port/protocol
          - 5671:5671/tcp   # AMQP SSL port
          - 5672:5672/tcp   # AMQP standard port
          - 15672:15672/tcp # Management: HTTP, CLI
          #- 15671:15671/tcp # Management: SSL port
          #- 25672:25672/tcp # inter-node or CLI
          #- 4369:4369/tcp   # epmd

    env:
      TASK: '${{ matrix.task }}'

      # We need to explicitly specify terminal width otherwise some CLI tests fail on container
      # environments where small terminal size is used.
      COLUMNS: '120'
      PYLINT_CONCURRENCY: '2'

      # CI st2.conf (with ST2_CI_USER user instead of stanley)
      ST2_CONF: 'conf/st2.ci.conf'

      # Tell StackStorm that we are indeed in CI mode, previously we hard coded a Travis specific
      # environment variable in our test code, making it a PITA when we switch CI providers.
      # Now, we simply set this environment varible here in the CI portion of our testing and
      # it avoids any CI provider type lock-in.
      ST2_CI: 'true'

      # Name of the user who is running the CI (on GitHub Actions this is 'runner')
      ST2_CI_USER: 'runner'
    steps:
      - name: Custom Environment Setup
        # built-in GitHub Actions environment variables
        # https://docs.github.com/en/free-pro-team@latest/actions/reference/environment-variables
        #
        # setting environment variables, so we can use shell logic
        # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
        run: |
          IS_NIGHTLY_BUILD=$([ "${GITHUB_EVENT_NAME}" = "schedule" ] && echo "yes" || echo "no")
          echo "IS_NIGHTLY_BUILD=${IS_NIGHTLY_BUILD}" >> $GITHUB_ENV

          # NOTE: We only enable coverage for master builds and not pull requests
          # since it has huge performance overhead (tests are 50% or so slower)
          ENABLE_COVERAGE=$([ "${GITHUB_EVENT_NAME}" != "pull_request" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ] && echo "yes" || echo "no")
          echo "ENABLE_COVERAGE=${ENABLE_COVERAGE}" >> $GITHUB_ENV

          # We only run tests with "--with-timer" flag on master and not for PRs since it adds 1-2
          # minutes of overhead to each build.
          NOSE_TIME=$([ "${GITHUB_EVENT_NAME}" != "pull_request" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ] && echo "yes" || echo "no")
          echo "NOSE_TIME=${NOSE_TIME}" >> $GITHUB_ENV

          # Setup the path to the st2 repo in the CI build system
          echo "ST2_CI_REPO_PATH=${GITHUB_WORKSPACE}" >> $GITHUB_ENV
      - name: Checkout repository
        uses: actions/checkout@v2
      - name: 'Set up Python (${{ matrix.python-version }})'
        uses: actions/setup-python@v2
        with:
          python-version: '${{ matrix.python-version }}'
      - name: Get date components for use in cache-keys
        id: date
        run: |
          echo "::set-output name=year::$(/bin/date -u "+%Y")"
          echo "::set-output name=month::$(/bin/date -u "+%m")"
          echo "::set-output name=week::$(/bin/date -u "+%U")"
      - uses: actions/cache@v2
        with:
          path: |
            ~/.cache/pip
            virtualenv
            ~/virtualenv
          # TODO: maybe make the virtualenv a partial cache to exclude st2*?
          # !virtualenv/lib/python*/site-packages/st2*
          # !virtualenv/bin/st2*
          key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-python-${{ matrix.python }}-
      - uses: actions/cache@v2
        with:
          path: |
            /var/cache/apt/archives/*.deb
            /var/cache/apt/archives/partial/*.deb
            /var/cache/apt/*.bin
          key: ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
          restore-keys: |
            ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-
            ${{ runner.os }}-apt-
      - name: Install apt depedencies
        run: |
          # install dev dependencies for Python YAML and LDAP packages
          # https://github.com/StackStorm/st2-auth-ldap
          sudo apt-get -y update
          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
      - name: Install virtualenv
        run: |
          # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
          #       only have to update it one place when we change the version
          # Note: Use --user to avoid polluting system site-packages (which breaks one of our tests)
          # TODO: simplify this once fixed in contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
          if [[ ! -f ~/virtualenv/bin/virtualenv ]]; then  # use the cached version whenever possible
            pip install --user --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
            virtualenv --no-download ~/virtualenv
            ~/virtualenv/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
            # drop the --user install virtualenv to prevent polluting tests
            pip freeze --user | xargs pip uninstall -y
            mkdir -p ~/.local/bin
            ln -s ~/virtualenv/bin/virtualenv ~/.local/bin/virtualenv
          fi
          which virtualenv
          virtualenv --version
      - name: Install requirements
        run: |
          ./scripts/travis/install-requirements.sh
      - name: Setup integration tests
        run: |
          # prep a ci-specific dev conf file that uses runner instead of stanley
          # this user is the username of the user in GitHub actions, used for SSH, etc during
          # integration tests (important)
          cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
          scripts/travis/add-itest-user-key.sh
          sudo .circle/add-itest-user.sh
      - name: Permissions Workaround
        if: "${{ env.TASK == 'ci-packs-tests' || env.TASK == 'ci-integration' }}"
        run: |
          echo "$ST2_CI_REPO_PATH"
          sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/travis/permissions-workaround.sh
      - name: Reconfigure RabbitMQ
        if: "${{ env.TASK == 'ci-unit' || env.TASK == 'ci-integration' }}"
        # bitnami image allows (see bitnami/rabbitmq readme):
        # Here we're copying a rabbitmq.config file which won't do anything.
        # We need to switch to custom.conf or advanced.config.<commit_msg>Handle restored ~/virtualenv cache correctly<commit_after>name: ci

on:
  push:
    branches:
      # only on merges to master branch
      - master
      # and version branches, which only include minor versions (eg: v3.4)
      - v[0-9]+.[0-9]+
    tags:
      # also version tags, which include bugfix releases (eg: v3.4.0)
      - v[0-9]+.[0-9]+.[0-9]+
  pull_request:
    type: [opened, reopened, edited]
    branches:
      # Only for PRs targeting those branches
      - master
      - v[0-9]+.[0-9]+
  schedule:
    # run every night at midnight
    - cron:  '0 0 * * *'

jobs:
  ci:
    name: '${{ matrix.name }} - python (${{ matrix.python-version }})'
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: 'Lint Checks'
            task: 'ci-checks'
            python-version: '3.6'
          - name: 'Compile'
            task: 'ci-compile'
            python-version: '3.6'
          - name: 'Pack Tests'
            task: 'ci-packs-tests'
            python-version: '3.6'
          - name: 'Unit Tests'
            task: 'ci-unit'
            python-version: '3.6'
          # This job is slow so we only run in on a daily basis
          # - name: 'Micro Benchmarks'
          #   task: 'micro-benchmarks'
          #   python-version: '3.6'
          - name: 'Integration Tests'
            task: 'ci-integration'
            python-version: '3.6'
    services:
      mongo:
        image: mongo:4.0
        ports:
          - 27017:27017

      # In GHA, these services are started first before the code is checked out.
      # We use bitnami images to facilitate reconfiguring RabbitMQ during ci-integration tests.
      # We rely on custom config and SSL certs that are in the repo.
      # Many images require config in env vars (which we can't change during the test job)
      # or they require config in entrypoint args (which we can't override for GHA services)
      # bitnami builds ways to get config files from mounted volumes.
      rabbitmq:
        image: bitnami/rabbitmq:3.8
        volumes:
          - /home/runner/rabbitmq_conf:/bitnami/conf  # RABBITMQ_MOUNTED_CONF_DIR
        env:
          # tell bitnami/rabbitmq to enable this by default
          RABBITMQ_PLUGINS: rabbitmq_management
          RABBITMQ_USERNAME: guest
          RABBITMQ_PASSWORD: <PASSWORD>

        # These are strictly docker options, not entrypoint args (GHA restriction)
        options: >-
          --name rabbitmq

        ports:
          # These 6 ports are exposed by bitnami/rabbitmq (see https://www.rabbitmq.com/networking.html#ports)
          # host_port:container_port/protocol
          - 5671:5671/tcp   # AMQP SSL port
          - 5672:5672/tcp   # AMQP standard port
          - 15672:15672/tcp # Management: HTTP, CLI
          #- 15671:15671/tcp # Management: SSL port
          #- 25672:25672/tcp # inter-node or CLI
          #- 4369:4369/tcp   # epmd

    env:
      TASK: '${{ matrix.task }}'

      # We need to explicitly specify terminal width otherwise some CLI tests fail on container
      # environments where small terminal size is used.
      COLUMNS: '120'
      PYLINT_CONCURRENCY: '2'

      # CI st2.conf (with ST2_CI_USER user instead of stanley)
      ST2_CONF: 'conf/st2.ci.conf'

      # Tell StackStorm that we are indeed in CI mode, previously we hard coded a Travis specific
      # environment variable in our test code, making it a PITA when we switch CI providers.
      # Now, we simply set this environment varible here in the CI portion of our testing and
      # it avoids any CI provider type lock-in.
      ST2_CI: 'true'

      # Name of the user who is running the CI (on GitHub Actions this is 'runner')
      ST2_CI_USER: 'runner'
    steps:
      - name: Custom Environment Setup
        # built-in GitHub Actions environment variables
        # https://docs.github.com/en/free-pro-team@latest/actions/reference/environment-variables
        #
        # setting environment variables, so we can use shell logic
        # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
        run: |
          IS_NIGHTLY_BUILD=$([ "${GITHUB_EVENT_NAME}" = "schedule" ] && echo "yes" || echo "no")
          echo "IS_NIGHTLY_BUILD=${IS_NIGHTLY_BUILD}" >> $GITHUB_ENV

          # NOTE: We only enable coverage for master builds and not pull requests
          # since it has huge performance overhead (tests are 50% or so slower)
          ENABLE_COVERAGE=$([ "${GITHUB_EVENT_NAME}" != "pull_request" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ] && echo "yes" || echo "no")
          echo "ENABLE_COVERAGE=${ENABLE_COVERAGE}" >> $GITHUB_ENV

          # We only run tests with "--with-timer" flag on master and not for PRs since it adds 1-2
          # minutes of overhead to each build.
          NOSE_TIME=$([ "${GITHUB_EVENT_NAME}" != "pull_request" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ] && echo "yes" || echo "no")
          echo "NOSE_TIME=${NOSE_TIME}" >> $GITHUB_ENV

          # Setup the path to the st2 repo in the CI build system
          echo "ST2_CI_REPO_PATH=${GITHUB_WORKSPACE}" >> $GITHUB_ENV
      - name: Checkout repository
        uses: actions/checkout@v2
      - name: 'Set up Python (${{ matrix.python-version }})'
        uses: actions/setup-python@v2
        with:
          python-version: '${{ matrix.python-version }}'
      - name: Get date components for use in cache-keys
        id: date
        run: |
          echo "::set-output name=year::$(/bin/date -u "+%Y")"
          echo "::set-output name=month::$(/bin/date -u "+%m")"
          echo "::set-output name=week::$(/bin/date -u "+%U")"
      - uses: actions/cache@v2
        with:
          path: |
            ~/.cache/pip
            virtualenv
            ~/virtualenv
          # TODO: maybe make the virtualenv a partial cache to exclude st2*?
          # !virtualenv/lib/python*/site-packages/st2*
          # !virtualenv/bin/st2*
          key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-python-${{ matrix.python }}-
      - uses: actions/cache@v2
        with:
          path: |
            /var/cache/apt/archives/*.deb
            /var/cache/apt/archives/partial/*.deb
            /var/cache/apt/*.bin
          key: ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
          restore-keys: |
            ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-
            ${{ runner.os }}-apt-
      - name: Install apt depedencies
        run: |
          # install dev dependencies for Python YAML and LDAP packages
          # https://github.com/StackStorm/st2-auth-ldap
          sudo apt-get -y update
          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
      - name: Install virtualenv
        run: |
          set -x
          # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
          #       only have to update it one place when we change the version
          # Note: Use --user to avoid polluting system site-packages (which breaks one of our tests)
          # TODO: simplify this once fixed in contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
          if [[ ! -f ~/virtualenv/bin/virtualenv ]]; then  # use the cached version whenever possible
            pip install --user --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
            virtualenv --no-download ~/virtualenv
            ~/virtualenv/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
            # drop the --user install virtualenv to prevent polluting tests
            pip freeze --user | xargs pip uninstall -y
          fi
          mkdir -p ~/.local/bin
          ln -s ~/virtualenv/bin/virtualenv ~/.local/bin/virtualenv
          which virtualenv
          virtualenv --version
      - name: Install requirements
        run: |
          ./scripts/travis/install-requirements.sh
      - name: Setup integration tests
        run: |
          # prep a ci-specific dev conf file that uses runner instead of stanley
          # this user is the username of the user in GitHub actions, used for SSH, etc during
          # integration tests (important)
          cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
          scripts/travis/add-itest-user-key.sh
          sudo .circle/add-itest-user.sh
      - name: Permissions Workaround
        if: "${{ env.TASK == 'ci-packs-tests' || env.TASK == 'ci-integration' }}"
        run: |
          echo "$ST2_CI_REPO_PATH"
          sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/travis/permissions-workaround.sh
      - name: Reconfigure RabbitMQ
        if: "${{ env.TASK == 'ci-unit' || env.TASK == 'ci-integration' }}"
        # bitnami image allows (see bitnami/rabbitmq readme):
        # Here we're copying a rabbitmq.config file which won't do anything.
        # We need to switch to custom.conf or advanced.config.
{'fraction_non_alphanumeric': 0.11390822096563723, 'fraction_numerical': 0.01468029578077425, 'mean_word_length': 2.087628000671479, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4706247', 'n_tokens_mistral': 5762, 'n_tokens_neox': 5463, 'n_words': 1647}