<commit_before># This file is responsible for configuring the `great_expectations` pipeline (https://dev.azure.com/great-expectations/great_expectations/_build)
#
# The pipeline is run under the following conditions:
#   - On the main branch when a weekly release is being cut
#   - On the develop branch as scheduled by the below cron job
#
#  `great_expectations` runs the entire test suite and is meant to safeguard the codebase against any errors that may have slipped through the cracks
#  from `dependency_graph`. Additionally, it is responsible for deployment during the weekly release cut.

schedules:
- cron: 0 */3 * * *
  displayName: Scheduled Runs
  branches:
    include:
    - develop
    exclude:
    - main
  always: false # Will only trigger if the state of the codebase has changed sinced the last scheduled run

trigger:
  branches:
    include:
    - pre_pr-* # Can be used to test both `great_expectations` and `dependency_graph` pipelines
    - main
    exclude:
    - develop

resources:
  containers:
  - container: postgres
    image: postgres:11
    ports:
    - 5432:5432
    env:
      POSTGRES_DB: "test_ci"
      POSTGRES_HOST_AUTH_METHOD: "trust"
  - container: mysql
    image: mysql:8.0.20
    ports:
      - 3306:3306
    env:
      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
      MYSQL_DATABASE: test_ci
  - container: mssql
    image: mcr.microsoft.com/mssql/server:2019-latest
    env:
      ACCEPT_EULA: Y
      MSSQL_SA_PASSWORD: <PASSWORD>%^&*
      MSSQL_DB: test_ci
      MSSQL_PID: Developer
    ports:
      - 1433:1433

# The pipeline is run under two primary conditions: if merging into main or as scheduled by the above cron job.
variables:
  isMain: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')]
  isScheduled: $[and(eq(variables['Build.SourceBranch'], 'refs/heads/develop'), eq(variables['Build.Reason'], 'Schedule'))]
  isManual: $[eq(variables['Build.Reason'], 'Manual')]
  GE_USAGE_STATISTICS_URL: "https://qa.stats.greatexpectations.io/great_expectations/v1/usage_statistics"

stages:
  - stage: lint
    pool:
      vmImage: 'ubuntu-latest'

    jobs:
      - job: lint
        condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))
        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: 3.7
            displayName: 'Use Python 3.7'

          - script: |
              pip install isort[requirements]==5.4.2 flake8==3.8.3 black==22.3.0 pyupgrade==2.7.2
              EXIT_STATUS=0
              isort . --check-only --skip docs/ || EXIT_STATUS=$?
              black --check --exclude docs/ . || EXIT_STATUS=$?
              flake8 great_expectations/core || EXIT_STATUS=$?
              pyupgrade --py3-plus || EXIT_STATUS=$?
              exit $EXIT_STATUS

  - stage: custom_checks
    pool:
      vmImage: 'ubuntu-latest'

    jobs:
    - job: type_hint_checker
      condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))
      steps:
      - script: |
          pip install mypy # Prereq for type hint script
          python scripts/check_type_hint_coverage.py
        name: TypeHintChecker

    - job: docstring_checker
      condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))
      steps:
      - bash: python scripts/check_docstring_coverage.py
        name: DocstringChecker

  - stage: import_ge
    dependsOn: [lint]
    pool:
      vmImage: 'ubuntu-18.04'
    jobs:
      - job: import_ge
        condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))

        strategy:
         matrix:
           Python37:
             python.version: '3.7'
           Python38:
             python.version: '3.8'
           Python39:
             python.version: '3.9'


        steps:
         - task: UsePythonVersion@0
           inputs:
             versionSpec: '$(python.version)'
           displayName: 'Use Python $(python.version)'

         - bash: python -m pip install --upgrade pip==21.3.1
           displayName: 'Update pip'

         - script: |
             pip install  .
           displayName: 'Install GE and required dependencies (i.e. not sqlalchemy)'

         - script: |
             python -c "import great_expectations as ge; print('Successfully imported GE Version:', ge.__version__)"
           displayName: 'Import Great Expectations'

  - stage: required
    dependsOn: [lint, import_ge, custom_checks]
    pool:
      vmImage: 'ubuntu-18.04'

    jobs:
      - job: compatibility_matrix
        condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))
        variables:
          GE_pytest_opts: '--no-sqlalchemy'
        strategy:
          matrix:
            Python37-Pandas023:
              python.version: '3.7'
              numpy.version: '1.17.0'
              pandas.version: '0.23.4'
              scipy.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev-test.txt --constraint constraints-dev.txt'
            Python37-Pandas025:
              python.version: '3.7'
              numpy.version: '1.17.0'
              pandas.version: '0.25.3'
              # numpy 1.20 and pandas 0.25.3 do not coexist happily
              scipy.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev-test.txt --constraint constraints-dev.txt'
            Python38-PandasLatest:
              python.version: '3.8'
              numpy.version: 'latest'
              pandas.version: 'latest'
              scipy.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev-test.txt --constraint constraints-dev.txt'

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip==21.3.1
            displayName: 'Update pip'

          - script: |
              if [[ $(numpy.version) == 'latest' ]]; then
                pip install numpy
              else
                pip install numpy==$(numpy.version)
              fi
            displayName: 'Install numpy'

          - script: |
              if [[ $(pandas.version) == 'latest' ]]; then
                pip install pandas
              else
                pip install pandas==$(pandas.version)
              fi
            displayName: 'Install pandas'

          - script: |
              if [[ $(scipy.version) == 'latest' ]]; then
                pip install scipy
              else
                pip install scipy==$(scipy.version)
              fi
            displayName: 'Install scipy'

          - script: |
              pip install $(GE_pytest_pip_opts)
              pip install  --requirement requirements.txt
              # Consider fragmenting *all* integration tests into separate folder and run
              pip install  .
            displayName: 'Install dependencies'

          - script: |
              pip install pytest pytest-cov pytest-azurepipelines
              pytest $(GE_pytest_opts) --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html --ignore=tests/cli --ignore=tests/integration/usage_statistics
            displayName: 'pytest'

          - task: PublishTestResults@2
            condition: succeededOrFailed()
            inputs:
              testResultsFiles: '**/test-*.xml'
              testRunTitle: 'Publish test results for Python $(python.version)'

          - task: PublishCodeCoverageResults@1
            inputs:
              codeCoverageTool: Cobertura
              summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
              reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'

      - job: comprehensive
        condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))

        services:
          postgres: postgres

        variables:
          GE_pytest_opts: '--postgresql --spark'

        strategy:
          matrix:
            Python37:
              python.version: '3.7'
              pandas.version: 'latest'
              GE_pytest_pip_opts: '"pyspark<3.0.0" --requirement requirements-dev.txt --constraint constraints-dev.txt'
            Python38:
              python.version: '3.8'
              pandas.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev.txt --constraint constraints-dev.txt'
            Python39:
              python.version: '3.9'
              pandas.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev.txt --constraint constraints-dev.txt'

        steps:<commit_msg>[MAINTENANCE] Add timeout to great_expectations pipeline stages to prevent false positive build failures<commit_after># This file is responsible for configuring the `great_expectations` pipeline (https://dev.azure.com/great-expectations/great_expectations/_build)
#
# The pipeline is run under the following conditions:
#   - On the main branch when a weekly release is being cut
#   - On the develop branch as scheduled by the below cron job
#
#  `great_expectations` runs the entire test suite and is meant to safeguard the codebase against any errors that may have slipped through the cracks
#  from `dependency_graph`. Additionally, it is responsible for deployment during the weekly release cut.

schedules:
- cron: 0 */3 * * *
  displayName: Scheduled Runs
  branches:
    include:
    - develop
    exclude:
    - main
  always: false # Will only trigger if the state of the codebase has changed sinced the last scheduled run

trigger:
  branches:
    include:
    - pre_pr-* # Can be used to test both `great_expectations` and `dependency_graph` pipelines
    - main
    exclude:
    - develop

resources:
  containers:
  - container: postgres
    image: postgres:11
    ports:
    - 5432:5432
    env:
      POSTGRES_DB: "test_ci"
      POSTGRES_HOST_AUTH_METHOD: "trust"
  - container: mysql
    image: mysql:8.0.20
    ports:
      - 3306:3306
    env:
      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
      MYSQL_DATABASE: test_ci
  - container: mssql
    image: mcr.microsoft.com/mssql/server:2019-latest
    env:
      ACCEPT_EULA: Y
      MSSQL_SA_PASSWORD: <PASSWORD>%^&*
      MSSQL_DB: test_ci
      MSSQL_PID: Developer
    ports:
      - 1433:1433

# The pipeline is run under two primary conditions: if merging into main or as scheduled by the above cron job.
variables:
  isMain: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')]
  isScheduled: $[and(eq(variables['Build.SourceBranch'], 'refs/heads/develop'), eq(variables['Build.Reason'], 'Schedule'))]
  isManual: $[eq(variables['Build.Reason'], 'Manual')]
  GE_USAGE_STATISTICS_URL: "https://qa.stats.greatexpectations.io/great_expectations/v1/usage_statistics"

stages:
  - stage: lint
    pool:
      vmImage: 'ubuntu-latest'

    jobs:
      - job: lint
        condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))
        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: 3.7
            displayName: 'Use Python 3.7'

          - script: |
              pip install isort[requirements]==5.4.2 flake8==3.8.3 black==22.3.0 pyupgrade==2.7.2
              EXIT_STATUS=0
              isort . --check-only --skip docs/ || EXIT_STATUS=$?
              black --check --exclude docs/ . || EXIT_STATUS=$?
              flake8 great_expectations/core || EXIT_STATUS=$?
              pyupgrade --py3-plus || EXIT_STATUS=$?
              exit $EXIT_STATUS

  - stage: custom_checks
    pool:
      vmImage: 'ubuntu-latest'

    jobs:
    - job: type_hint_checker
      condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))
      steps:
      - script: |
          pip install mypy # Prereq for type hint script
          python scripts/check_type_hint_coverage.py
        name: TypeHintChecker

    - job: docstring_checker
      condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))
      steps:
      - bash: python scripts/check_docstring_coverage.py
        name: DocstringChecker

  - stage: import_ge
    dependsOn: [lint]
    pool:
      vmImage: 'ubuntu-18.04'
    jobs:
      - job: import_ge
        condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))

        strategy:
         matrix:
           Python37:
             python.version: '3.7'
           Python38:
             python.version: '3.8'
           Python39:
             python.version: '3.9'


        steps:
         - task: UsePythonVersion@0
           inputs:
             versionSpec: '$(python.version)'
           displayName: 'Use Python $(python.version)'

         - bash: python -m pip install --upgrade pip==21.3.1
           displayName: 'Update pip'

         - script: |
             pip install  .
           displayName: 'Install GE and required dependencies (i.e. not sqlalchemy)'

         - script: |
             python -c "import great_expectations as ge; print('Successfully imported GE Version:', ge.__version__)"
           displayName: 'Import Great Expectations'

  - stage: required
    dependsOn: [lint, import_ge, custom_checks]
    pool:
      vmImage: 'ubuntu-18.04'

    jobs:
      - job: compatibility_matrix
        condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))
        timeoutInMinutes: 75
        variables:
          GE_pytest_opts: '--no-sqlalchemy'
        strategy:
          matrix:
            Python37-Pandas023:
              python.version: '3.7'
              numpy.version: '1.17.0'
              pandas.version: '0.23.4'
              scipy.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev-test.txt --constraint constraints-dev.txt'
            Python37-Pandas025:
              python.version: '3.7'
              numpy.version: '1.17.0'
              pandas.version: '0.25.3'
              # numpy 1.20 and pandas 0.25.3 do not coexist happily
              scipy.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev-test.txt --constraint constraints-dev.txt'
            Python38-PandasLatest:
              python.version: '3.8'
              numpy.version: 'latest'
              pandas.version: 'latest'
              scipy.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev-test.txt --constraint constraints-dev.txt'

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip==21.3.1
            displayName: 'Update pip'

          - script: |
              if [[ $(numpy.version) == 'latest' ]]; then
                pip install numpy
              else
                pip install numpy==$(numpy.version)
              fi
            displayName: 'Install numpy'

          - script: |
              if [[ $(pandas.version) == 'latest' ]]; then
                pip install pandas
              else
                pip install pandas==$(pandas.version)
              fi
            displayName: 'Install pandas'

          - script: |
              if [[ $(scipy.version) == 'latest' ]]; then
                pip install scipy
              else
                pip install scipy==$(scipy.version)
              fi
            displayName: 'Install scipy'

          - script: |
              pip install $(GE_pytest_pip_opts)
              pip install  --requirement requirements.txt
              # Consider fragmenting *all* integration tests into separate folder and run
              pip install  .
            displayName: 'Install dependencies'

          - script: |
              pip install pytest pytest-cov pytest-azurepipelines
              pytest $(GE_pytest_opts) --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html --ignore=tests/cli --ignore=tests/integration/usage_statistics
            displayName: 'pytest'

          - task: PublishTestResults@2
            condition: succeededOrFailed()
            inputs:
              testResultsFiles: '**/test-*.xml'
              testRunTitle: 'Publish test results for Python $(python.version)'

          - task: PublishCodeCoverageResults@1
            inputs:
              codeCoverageTool: Cobertura
              summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
              reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'

      - job: comprehensive
        condition: or(eq(variables.isScheduled, true), eq(variables.isMain, true), eq(variables.isManual, true))
        timeoutInMinutes: 75

        services:
          postgres: postgres

        variables:
          GE_pytest_opts: '--postgresql --spark'

        strategy:
          matrix:
            Python37:
              python.version: '3.7'
              pandas.version: 'latest'
              GE_pytest_pip_opts: '"pyspark<3.0.0" --requirement requirements-dev.txt --constraint constraints-dev.txt'
            Python38:
              python.version: '3.8'
              pandas.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev.txt --constraint constraints-dev.txt'
            Python39:
              python.version: '3.9'
              pandas.version: 'latest'
              GE_pytest_pip_opts: '--requirement requirements-dev.txt --constraint constraints-dev.txt'

        steps:
{'fraction_non_alphanumeric': 0.10330370618816032, 'fraction_numerical': 0.016658281625579967, 'mean_word_length': 2.0153379403337266, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1504595', 'n_tokens_mistral': 5407, 'n_tokens_neox': 5148, 'n_words': 1265}