<commit_before>## ======================================================================
## groups
## ======================================================================

groups:
- name: all
  jobs:
  - validate_pipeline
################################
  - gate_compile_start
  - compile_gpdb_centos6
  - compile_gpdb_open_source_centos6
  - compile_gpdb_binary_swap_centos6
  - compile_gpdb_centos7
  - compile_gpdb_sles11
  - compile_gpdb_ubuntu16
  - compile_gpdb_windows_cl
  - compile_gpdb_aix7_remote
  - gpdb_rc_packaging_centos
  - client_loader_remote_test_aix
  - gate_compile_end
################################
  - gate_icw_start
  - icw_planner_centos6
  - icw_gporca_centos6
  - icw_gporca_centos7
  - icw_gporca_sles11
  - icw_planner_ubuntu16
  - icw_planner_ictcp_centos6
  - icw_gporca_centos6_gpos_memory
  - fts
  - storage
  - gate_icw_end
################################
  - gate_cs_misc_start
  - cs_walrep_1
  - cs_walrep_2
  - cs_pg_twophase_01_10
  - cs_pg_twophase_11_20
  - cs_pg_twophase_21_30
  - cs_pg_twophase_31_40
  - cs_pg_twophase_41_49
  - cs_pg_twophase_switch_01_12
  - cs_pg_twophase_switch_13_24
  - cs_pg_twophase_switch_25_33
  - gate_cs_misc_end
################################
  - gate_cluster_start
  - mpp_interconnect
  - DPM_gptransfer-5x-to-5x
  - cs-filerep-schema-topology-crashrecov
  - DPM_backup-restore
  - gate_cluster_end
################################
  - gate_mm_misc_start
  - MM_gppkg
  - MM_gprecoverseg
  - MM_gpcheck
  - MM_analyzedb
  - MM_gpperfmon
  - MM_pt-rebuild
  - MM_gpinitsystem
  - MU_check_centos
  - MM_gpinitstandby
  - gate_mm_misc_end
################################
  - gate_nightly_start
  - DPM_backup_43_restore_5
  - MM_gpexpand
  - cs-aoco-compression
  - DPM_gptransfer-43x-to-5x
  - gate_nightly_end
################################
  - gate_general_misc_start
  - MM_gpcheckcat
  - QP_runaway-query
  - QP_memory-accounting
  - QP_optimizer-functional
  - regression_tests_pxf_hdp
  - regression_tests_pxf_cdh
  - regression_tests_gphdfs_centos
  - regression_tests_gpcloud_centos
  - mpp_resource_group_centos6
  - gate_general_misc_end
################################
  - gate_filerep_start
  - cs_filerep_e2e_full_mirror
  - cs_filerep_e2e_incr_mirror
  - cs_filerep_e2e_full_primary
  - cs_filerep_e2e_incr_primary
  - gate_filerep_end
################################

- name: Release
  jobs:
  - Release_Candidate

- name: Remaining Pulse
  jobs:
  - MM_gpcheckcat
  - MM_gpexpand
  - DPM_gptransfer-43x-to-5x
  - DPM_gptransfer-5x-to-5x
  - cs-filerep-schema-topology-crashrecov
  - cs-aoco-compression
  - mpp_interconnect

- name: Adopted CCP
  jobs:
  - cs_walrep_1
  - cs_walrep_2
  - mpp_resource_group_centos6
  - cs_filerep_e2e_full_primary
  - cs_filerep_e2e_incr_primary
  - cs_filerep_e2e_full_mirror
  - cs_filerep_e2e_incr_mirror
  - DPM_backup-restore
  - MM_gpcheckcat
  - MM_gppkg
  - MM_gprecoverseg
  - DPM_backup_43_restore_5

- name: Experimental Tests
  jobs:
  - icw_gporca_centos6_gpos_memory

- name: G:Compile
  jobs:
  - gate_compile_start
  - compile_gpdb_centos6
  - compile_gpdb_open_source_centos6
  - compile_gpdb_binary_swap_centos6
  - compile_gpdb_centos7
  - compile_gpdb_sles11
  - compile_gpdb_ubuntu16
  - compile_gpdb_windows_cl
  - compile_gpdb_aix7_remote
  - gpdb_rc_packaging_centos
  - client_loader_remote_test_aix
  - gate_compile_end

- name: G:ICW
  jobs:
  - gate_icw_start
  - icw_planner_centos6
  - icw_planner_ubuntu16
  - icw_gporca_centos6
  - icw_gporca_centos7
  - icw_gporca_sles11
  - icw_planner_ictcp_centos6
  - icw_gporca_centos6_gpos_memory
  - fts
  - storage
  - gate_icw_end

- name: G:CS_Misc
  jobs:
  - gate_cs_misc_start
  - cs_walrep_1
  - cs_walrep_2
  - cs_pg_twophase_01_10
  - cs_pg_twophase_11_20
  - cs_pg_twophase_21_30
  - cs_pg_twophase_31_40
  - cs_pg_twophase_41_49
  - cs_pg_twophase_switch_01_12
  - cs_pg_twophase_switch_13_24
  - cs_pg_twophase_switch_25_33
  - gate_cs_misc_end

- name: G:Cluster
  jobs:
  - gate_cluster_start
  - mpp_interconnect
  - DPM_gptransfer-5x-to-5x
  - cs-filerep-schema-topology-crashrecov
  - DPM_backup-restore
  - gate_cluster_end

- name: G:MM_Misc
  jobs:
  - gate_mm_misc_start
  - MM_gppkg
  - MM_gprecoverseg
  - MM_gpcheck
  - MM_analyzedb
  - MM_gpperfmon
  - MM_pt-rebuild
  - MM_gpinitsystem
  - MU_check_centos
  - MM_gpinitstandby
  - gate_mm_misc_end

- name: G:Nightly
  jobs:
  - gate_nightly_start
  - DPM_backup_43_restore_5
  - MM_gpexpand
  - cs-aoco-compression
  - DPM_gptransfer-43x-to-5x
  - gate_nightly_end

- name: G:General
  jobs:
  - gate_general_misc_start
  - MM_gpcheckcat
  - QP_runaway-query
  - QP_memory-accounting
  - QP_optimizer-functional
  - regression_tests_pxf_hdp
  - regression_tests_pxf_cdh
  - regression_tests_gphdfs_centos
  - regression_tests_gpcloud_centos
  - mpp_resource_group_centos6
  - gate_general_misc_end

- name: G:FileRep
  jobs:
  - gate_filerep_start
  - cs_filerep_e2e_full_mirror
  - cs_filerep_e2e_incr_mirror
  - cs_filerep_e2e_full_primary
  - cs_filerep_e2e_incr_primary
  - gate_filerep_end

## ======================================================================
## resource types
## ======================================================================

resource_types:
- name: terraform
  type: docker-image
  source:
    repository: ljfranklin/terraform-resource

## ======================================================================
## resources
## ======================================================================

resources:
- name: ccp_src
  type: git
  source:
    branch: {{ccp-git-branch}}
    private_key: {{ccp-git-key}}
    uri: {{ccp-git-remote}}
    tag_filter: 1.0.0

- name: terraform
  type: terraform
  source:
    env:
      AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
      AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}}
    storage:
      access_key_id: {{tf-machine-access-key-id}}
      secret_access_key: {{tf-machine-secret-access-key}}
      region_name: {{aws-region}}
      # This is not parameterized, on purpose. All tfstates will go to this spot,
      # and different teams will place there clusters' tfstate files under different paths
      bucket: gpdb5-pipeline-dynamic-terraform
      ###########################################################
      # Two conventions are allowed for bucket_path:            #
      # 1) Cluster are expected to be destroyed automatically   #
      #    Toolsmiths will reap old and orphaned clusters       #
      #                                                         #
      #      bucket_path: prod/[Pipeline Name]/                 #
      #                                                         #
      # 2) Long lived clusters for development.                 #
      #    The team that creates it is responsible for cluster  #
      #                                                         #
      #     bucket_path: dev/[Team Name]/                       #
      #                                                         #
      ###########################################################
      bucket_path: {{tf-bucket-path}}

- name: terraform_for_dpm
  type: terraform
  source:
    env:
      AWS_ACCESS_KEY_ID: {{gpdb4-tf-machine-access-key-id}}
      AWS_SECRET_ACCESS_KEY: {{gpdb4-tf-machine-secret-access-key}}
    storage:
      access_key_id: {{gpdb4-tf-machine-access-key-id}}
      secret_access_key: {{gpdb4-tf-machine-secret-access-key}}
      region_name: {{aws-region}}
      # This is not parameterized, on purpose. All tfstates will go to this spot,
      # and different teams will place there clusters' tfstate files under different paths
      bucket: pivotal-pa-toolsmiths-pipeline-dynamic-terraform
      bucket_path: {{tf-bucket-path}}

- name: aix_environments
  type: pool
  source:
    uri: {{concourse-resource-pools-git-remote}}
    branch: master
    pool: client_loader_remote_aix
    private_key: {{concourse-resource-pools-git-key}}

- name: gpdb_src
  type: git
  source:
    branch: {{gpdb-git-branch}}
    uri: {{gpdb-git-remote}}
    ignore_paths:
    - gpdb-doc/*
    - README*

- name: gpdb_src_binary_swap
  type: git
  source:
    branch: {{gpdb-binary-swap-git-tag}}
    uri: {{gpdb-binary-swap-git-remote}}
    ignore_paths:
    - gpdb-doc/*
    - README*

- name: gpaddon_src
  type: git
  source:
    branch: {{gpaddon-git-branch}}
    private_key: {{gpaddon-git-key}}
    uri: {{gpaddon-git-remote}}

- name: pxf_src
  type: git
  source:
    branch: {{pxf-git-branch}}
    tag_filter: {{pxf-git-filter}}
    private_key: {{pxf-git-key}}
    uri: {{pxf-git-remote}}

- name: pxf_automation_src
  type: git
  source:
    branch: {{pxf_automation-git-branch}}
    private_key: {{pxf_automation-git-key}}
    uri: {{pxf_automation-git-remote}}

- name: centos-gpdb-dev-6
  type: docker-image
  source:
    repository: pivotaldata/centos-gpdb-dev
    tag: '6-gcc6.2-llvm3.7'

- name: centos-gpdb-dev-7
  type: docker-image
  source:
    repository: pivotaldata/centos-gpdb-dev
    tag: 7-gcc6.2-llvm3.7

- name: ubuntu-gpdb-dev-16
  type: docker-image
  source:
    repository: pivotaldata/ubuntu-gpdb-dev
    tag: '16.04_gcc_6_3'

- name: centos-mingw
  type: docker-image
  source:
    repository: pivotaldata/centos-mingw

- name: bin_gpdb_centos6
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{bin_gpdb_centos_versioned_file}}

- name: compiled_bits_ubuntu16
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{compiled_bits_ubuntu16_versioned_file}}

- name: binary_swap_gpdb_centos6
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{binary_swap_gpdb_centos_versioned_file}}

- name: bin_gpdb_centos7
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{bin_gpdb_centos7_versioned_file}}

- name: bin_gpdb_sles11
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{bin_gpdb_sles11_versioned_file}}

- name: bin_gpdb_windows_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: windows-cl/greenplum-clients-5.(.*)-WinXP-x86_32.msi

- name: bin_gpdb_windows_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: windows-cl/greenplum-loaders-5.(.*)-WinXP-x86_32.msi

- name: bin_gpdb4_centos6
  type: s3
  source:
    access_key_id: {{gpdb4-bucket-access-key-id}}
    bucket: {{gpdb4-bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{gpdb4-bucket-secret-access-key}}
    versioned_file: bin_gpdb_centos/bin_gpdb.tar.gz

- name: installer_rhel6_gpdb_rc
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/software_only_installer/greenplum-db-(.*)-rhel6-x86_64.zip

- name: installer_rhel6_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-rhel6-x86_64.zip

- name: installer_rhel6_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-rhel6-x86_64.zip

- name: installer_rhel7_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-rhel7-x86_64.zip

- name: installer_rhel7_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-rhel7-x86_64.zip

- name: installer_aix7_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-aix7_ppc_64.zip

- name: installer_aix7_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-aix7_ppc_64.zip

- name: qautils_rhel6_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: deliverables/QAUtils-rhel6-x86_64.tar.gz

- name: installer_sles11_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-sles11-x86_64.zip

- name: installer_sles11_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-sles11-x86_64.zip

- name: gpdb_src_tinc_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-db-(.*)-src.tar.gz

- name: gpdb_src_behave_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-db-(.*)-behave.tar.gz

- name: singlecluster-HDP
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    secret_access_key: {{bucket-secret-access-key}}
    bucket: {{pxf-aws-bucket-name}}
    region_name: {{aws-region}}
    versioned_file: singlecluster-without-pxf/singlecluster-HDP.tar.gz

- name: singlecluster-CDH
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    secret_access_key: {{bucket-secret-access-key}}
    bucket: {{pxf-aws-bucket-name}}
    region_name: {{aws-region}}
    versioned_file: singlecluster-without-pxf/singlecluster-CDH.tar.gz

- name: nightly-trigger
  type: time
  source:
    location: America/Los_Angeles
    days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday]
    start: 6:00 AM
    stop: 7:00 AM

- name: reduced-frequency-trigger
  type: time
  source:
    location: America/Los_Angeles
    days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday]
    start: {{reduced-frequency-trigger-start}}
    stop: {{reduced-frequency-trigger-stop}}

## ======================================================================
## reusable anchors
## ======================================================================

ccp_create_params_anchor: &ccp_default_params
  action: create
  delete_on_failure: true
  generate_random_name: true
  terraform_source: ccp_src/aws/

ccp_vars_anchor: &ccp_default_vars
  aws_instance-node-instance_type: t2.medium
  aws_ebs_volume_type: standard
  platform: centos6

ccp_destroy_anchor: &ccp_destroy
  put: terraform
  tags: ["gpdb5_ccp_external_worker"]
  params:
    action: destroy
    env_name_file: terraform/name
    terraform_source: ccp_src/aws/
    vars:
      aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
      aws_ebs_volume_type: standard
  get_params:
    action: destroy

dpm_ccp_destroy_anchor: &dpm_ccp_destroy
  put: terraform_for_dpm
  params:
    action: destroy
    env_name_file: terraform_for_dpm/name
    terraform_source: ccp_src/aws-pivotal-pa-toolsmiths/
    vars:
      aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
  get_params:
    action: destroy

ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params
  AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
  AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}}
  AWS_DEFAULT_REGION: {{aws-region}}
  BUCKET_PATH: {{tf-bucket-path}}
  BUCKET_NAME: {{tf-bucket-name}}

debug_sleep_anchor: &debug_sleep
  do:
  - task: debug_sleep
    tags: ["gpdb5_ccp_external_worker"]
    config:
      platform: linux
      image_resource:
        type: docker-image
        source:
          repository: alpine
          tag: latest
      run:
        path: 'sh'
        args: ['-c', 'sleep 6h']
    ensure:
      <<: *ccp_destroy

dpm_debug_sleep_anchor: &dpm_debug_sleep
  do:
  - task: debug_sleep
    config:
      platform: linux
      image_resource:
        type: docker-image
        source:
          repository: alpine
          tag: latest
      run:
        path: 'sh'
        args: ['-c', 'sleep 6h']
    ensure:
      <<: *dpm_ccp_destroy

pulse_properties_anchor: &pulse_properties
      PULSE_URL: {{pulse_url}}
      PULSE_USERNAME: {{pulse_username}}
      PULSE_PASSWORD: {{pulse_password}}

## ======================================================================
## jobs
## ======================================================================

# Stage 1: Build and C Unit Tests

jobs:

- name: validate_pipeline
  plan:
  - aggregate:
    - get: gpdb_src
      trigger: true
    - get: centos-gpdb-dev-6
  - task: validate_pipeline
    file: gpdb_src/concourse/tasks/validate_pipeline.yml
    image: centos-gpdb-dev-6

################# Compile Group
- name: gate_compile_start
  plan:
  - aggregate:
    - get: gpdb_src<commit_msg>pipeline - Bump CCP version and remove unused anchors<commit_after>## ======================================================================
## groups
## ======================================================================

groups:
- name: all
  jobs:
  - validate_pipeline
################################
  - gate_compile_start
  - compile_gpdb_centos6
  - compile_gpdb_open_source_centos6
  - compile_gpdb_binary_swap_centos6
  - compile_gpdb_centos7
  - compile_gpdb_sles11
  - compile_gpdb_ubuntu16
  - compile_gpdb_windows_cl
  - compile_gpdb_aix7_remote
  - gpdb_rc_packaging_centos
  - client_loader_remote_test_aix
  - gate_compile_end
################################
  - gate_icw_start
  - icw_planner_centos6
  - icw_gporca_centos6
  - icw_gporca_centos7
  - icw_gporca_sles11
  - icw_planner_ubuntu16
  - icw_planner_ictcp_centos6
  - icw_gporca_centos6_gpos_memory
  - fts
  - storage
  - gate_icw_end
################################
  - gate_cs_misc_start
  - cs_walrep_1
  - cs_walrep_2
  - cs_pg_twophase_01_10
  - cs_pg_twophase_11_20
  - cs_pg_twophase_21_30
  - cs_pg_twophase_31_40
  - cs_pg_twophase_41_49
  - cs_pg_twophase_switch_01_12
  - cs_pg_twophase_switch_13_24
  - cs_pg_twophase_switch_25_33
  - gate_cs_misc_end
################################
  - gate_cluster_start
  - mpp_interconnect
  - DPM_gptransfer-5x-to-5x
  - cs-filerep-schema-topology-crashrecov
  - DPM_backup-restore
  - gate_cluster_end
################################
  - gate_mm_misc_start
  - MM_gppkg
  - MM_gprecoverseg
  - MM_gpcheck
  - MM_analyzedb
  - MM_gpperfmon
  - MM_pt-rebuild
  - MM_gpinitsystem
  - MU_check_centos
  - MM_gpinitstandby
  - gate_mm_misc_end
################################
  - gate_nightly_start
  - DPM_backup_43_restore_5
  - MM_gpexpand
  - cs-aoco-compression
  - DPM_gptransfer-43x-to-5x
  - gate_nightly_end
################################
  - gate_general_misc_start
  - MM_gpcheckcat
  - QP_runaway-query
  - QP_memory-accounting
  - QP_optimizer-functional
  - regression_tests_pxf_hdp
  - regression_tests_pxf_cdh
  - regression_tests_gphdfs_centos
  - regression_tests_gpcloud_centos
  - mpp_resource_group_centos6
  - gate_general_misc_end
################################
  - gate_filerep_start
  - cs_filerep_e2e_full_mirror
  - cs_filerep_e2e_incr_mirror
  - cs_filerep_e2e_full_primary
  - cs_filerep_e2e_incr_primary
  - gate_filerep_end
################################

- name: Release
  jobs:
  - Release_Candidate

- name: Remaining Pulse
  jobs:
  - MM_gpcheckcat
  - MM_gpexpand
  - DPM_gptransfer-43x-to-5x
  - DPM_gptransfer-5x-to-5x
  - cs-filerep-schema-topology-crashrecov
  - cs-aoco-compression
  - mpp_interconnect

- name: Adopted CCP
  jobs:
  - cs_walrep_1
  - cs_walrep_2
  - mpp_resource_group_centos6
  - cs_filerep_e2e_full_primary
  - cs_filerep_e2e_incr_primary
  - cs_filerep_e2e_full_mirror
  - cs_filerep_e2e_incr_mirror
  - DPM_backup-restore
  - MM_gpcheckcat
  - MM_gppkg
  - MM_gprecoverseg
  - DPM_backup_43_restore_5

- name: Experimental Tests
  jobs:
  - icw_gporca_centos6_gpos_memory

- name: G:Compile
  jobs:
  - gate_compile_start
  - compile_gpdb_centos6
  - compile_gpdb_open_source_centos6
  - compile_gpdb_binary_swap_centos6
  - compile_gpdb_centos7
  - compile_gpdb_sles11
  - compile_gpdb_ubuntu16
  - compile_gpdb_windows_cl
  - compile_gpdb_aix7_remote
  - gpdb_rc_packaging_centos
  - client_loader_remote_test_aix
  - gate_compile_end

- name: G:ICW
  jobs:
  - gate_icw_start
  - icw_planner_centos6
  - icw_planner_ubuntu16
  - icw_gporca_centos6
  - icw_gporca_centos7
  - icw_gporca_sles11
  - icw_planner_ictcp_centos6
  - icw_gporca_centos6_gpos_memory
  - fts
  - storage
  - gate_icw_end

- name: G:CS_Misc
  jobs:
  - gate_cs_misc_start
  - cs_walrep_1
  - cs_walrep_2
  - cs_pg_twophase_01_10
  - cs_pg_twophase_11_20
  - cs_pg_twophase_21_30
  - cs_pg_twophase_31_40
  - cs_pg_twophase_41_49
  - cs_pg_twophase_switch_01_12
  - cs_pg_twophase_switch_13_24
  - cs_pg_twophase_switch_25_33
  - gate_cs_misc_end

- name: G:Cluster
  jobs:
  - gate_cluster_start
  - mpp_interconnect
  - DPM_gptransfer-5x-to-5x
  - cs-filerep-schema-topology-crashrecov
  - DPM_backup-restore
  - gate_cluster_end

- name: G:MM_Misc
  jobs:
  - gate_mm_misc_start
  - MM_gppkg
  - MM_gprecoverseg
  - MM_gpcheck
  - MM_analyzedb
  - MM_gpperfmon
  - MM_pt-rebuild
  - MM_gpinitsystem
  - MU_check_centos
  - MM_gpinitstandby
  - gate_mm_misc_end

- name: G:Nightly
  jobs:
  - gate_nightly_start
  - DPM_backup_43_restore_5
  - MM_gpexpand
  - cs-aoco-compression
  - DPM_gptransfer-43x-to-5x
  - gate_nightly_end

- name: G:General
  jobs:
  - gate_general_misc_start
  - MM_gpcheckcat
  - QP_runaway-query
  - QP_memory-accounting
  - QP_optimizer-functional
  - regression_tests_pxf_hdp
  - regression_tests_pxf_cdh
  - regression_tests_gphdfs_centos
  - regression_tests_gpcloud_centos
  - mpp_resource_group_centos6
  - gate_general_misc_end

- name: G:FileRep
  jobs:
  - gate_filerep_start
  - cs_filerep_e2e_full_mirror
  - cs_filerep_e2e_incr_mirror
  - cs_filerep_e2e_full_primary
  - cs_filerep_e2e_incr_primary
  - gate_filerep_end

## ======================================================================
## resource types
## ======================================================================

resource_types:
- name: terraform
  type: docker-image
  source:
    repository: ljfranklin/terraform-resource

## ======================================================================
## resources
## ======================================================================

resources:
- name: ccp_src
  type: git
  source:
    branch: {{ccp-git-branch}}
    private_key: {{ccp-git-key}}
    uri: {{ccp-git-remote}}
    tag_filter: 1.0.1

- name: terraform
  type: terraform
  source:
    env:
      AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
      AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}}
    storage:
      access_key_id: {{tf-machine-access-key-id}}
      secret_access_key: {{tf-machine-secret-access-key}}
      region_name: {{aws-region}}
      # This is not parameterized, on purpose. All tfstates will go to this spot,
      # and different teams will place there clusters' tfstate files under different paths
      bucket: gpdb5-pipeline-dynamic-terraform
      ###########################################################
      # Two conventions are allowed for bucket_path:            #
      # 1) Cluster are expected to be destroyed automatically   #
      #    Toolsmiths will reap old and orphaned clusters       #
      #                                                         #
      #      bucket_path: prod/[Pipeline Name]/                 #
      #                                                         #
      # 2) Long lived clusters for development.                 #
      #    The team that creates it is responsible for cluster  #
      #                                                         #
      #     bucket_path: dev/[Team Name]/                       #
      #                                                         #
      ###########################################################
      bucket_path: {{tf-bucket-path}}

- name: aix_environments
  type: pool
  source:
    uri: {{concourse-resource-pools-git-remote}}
    branch: master
    pool: client_loader_remote_aix
    private_key: {{concourse-resource-pools-git-key}}

- name: gpdb_src
  type: git
  source:
    branch: {{gpdb-git-branch}}
    uri: {{gpdb-git-remote}}
    ignore_paths:
    - gpdb-doc/*
    - README*

- name: gpdb_src_binary_swap
  type: git
  source:
    branch: {{gpdb-binary-swap-git-tag}}
    uri: {{gpdb-binary-swap-git-remote}}
    ignore_paths:
    - gpdb-doc/*
    - README*

- name: gpaddon_src
  type: git
  source:
    branch: {{gpaddon-git-branch}}
    private_key: {{gpaddon-git-key}}
    uri: {{gpaddon-git-remote}}

- name: pxf_src
  type: git
  source:
    branch: {{pxf-git-branch}}
    tag_filter: {{pxf-git-filter}}
    private_key: {{pxf-git-key}}
    uri: {{pxf-git-remote}}

- name: pxf_automation_src
  type: git
  source:
    branch: {{pxf_automation-git-branch}}
    private_key: {{pxf_automation-git-key}}
    uri: {{pxf_automation-git-remote}}

- name: centos-gpdb-dev-6
  type: docker-image
  source:
    repository: pivotaldata/centos-gpdb-dev
    tag: '6-gcc6.2-llvm3.7'

- name: centos-gpdb-dev-7
  type: docker-image
  source:
    repository: pivotaldata/centos-gpdb-dev
    tag: 7-gcc6.2-llvm3.7

- name: ubuntu-gpdb-dev-16
  type: docker-image
  source:
    repository: pivotaldata/ubuntu-gpdb-dev
    tag: '16.04_gcc_6_3'

- name: centos-mingw
  type: docker-image
  source:
    repository: pivotaldata/centos-mingw

- name: bin_gpdb_centos6
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{bin_gpdb_centos_versioned_file}}

- name: compiled_bits_ubuntu16
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{compiled_bits_ubuntu16_versioned_file}}

- name: binary_swap_gpdb_centos6
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{binary_swap_gpdb_centos_versioned_file}}

- name: bin_gpdb_centos7
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{bin_gpdb_centos7_versioned_file}}

- name: bin_gpdb_sles11
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: {{bin_gpdb_sles11_versioned_file}}

- name: bin_gpdb_windows_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: windows-cl/greenplum-clients-5.(.*)-WinXP-x86_32.msi

- name: bin_gpdb_windows_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: windows-cl/greenplum-loaders-5.(.*)-WinXP-x86_32.msi

- name: bin_gpdb4_centos6
  type: s3
  source:
    access_key_id: {{gpdb4-bucket-access-key-id}}
    bucket: {{gpdb4-bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{gpdb4-bucket-secret-access-key}}
    versioned_file: bin_gpdb_centos/bin_gpdb.tar.gz

- name: installer_rhel6_gpdb_rc
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/software_only_installer/greenplum-db-(.*)-rhel6-x86_64.zip

- name: installer_rhel6_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-rhel6-x86_64.zip

- name: installer_rhel6_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-rhel6-x86_64.zip

- name: installer_rhel7_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-rhel7-x86_64.zip

- name: installer_rhel7_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-rhel7-x86_64.zip

- name: installer_aix7_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-aix7_ppc_64.zip

- name: installer_aix7_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-aix7_ppc_64.zip

- name: qautils_rhel6_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    versioned_file: deliverables/QAUtils-rhel6-x86_64.tar.gz

- name: installer_sles11_gpdb_clients
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-clients-5.(.*)-sles11-x86_64.zip

- name: installer_sles11_gpdb_loaders
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-loaders-5.(.*)-sles11-x86_64.zip

- name: gpdb_src_tinc_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-db-(.*)-src.tar.gz

- name: gpdb_src_behave_tarball
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    bucket: {{bucket-name}}
    region_name: {{aws-region}}
    secret_access_key: {{bucket-secret-access-key}}
    regexp: deliverables/greenplum-db-(.*)-behave.tar.gz

- name: singlecluster-HDP
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    secret_access_key: {{bucket-secret-access-key}}
    bucket: {{pxf-aws-bucket-name}}
    region_name: {{aws-region}}
    versioned_file: singlecluster-without-pxf/singlecluster-HDP.tar.gz

- name: singlecluster-CDH
  type: s3
  source:
    access_key_id: {{bucket-access-key-id}}
    secret_access_key: {{bucket-secret-access-key}}
    bucket: {{pxf-aws-bucket-name}}
    region_name: {{aws-region}}
    versioned_file: singlecluster-without-pxf/singlecluster-CDH.tar.gz

- name: nightly-trigger
  type: time
  source:
    location: America/Los_Angeles
    days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday]
    start: 6:00 AM
    stop: 7:00 AM

- name: reduced-frequency-trigger
  type: time
  source:
    location: America/Los_Angeles
    days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday]
    start: {{reduced-frequency-trigger-start}}
    stop: {{reduced-frequency-trigger-stop}}

## ======================================================================
## reusable anchors
## ======================================================================

ccp_create_params_anchor: &ccp_default_params
  action: create
  delete_on_failure: true
  generate_random_name: true
  terraform_source: ccp_src/aws/

ccp_vars_anchor: &ccp_default_vars
  aws_instance-node-instance_type: t2.medium
  aws_ebs_volume_type: standard
  platform: centos6

ccp_destroy_anchor: &ccp_destroy
  put: terraform
  tags: ["gpdb5_ccp_external_worker"]
  params:
    action: destroy
    env_name_file: terraform/name
    terraform_source: ccp_src/aws/
    vars:
      aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
      aws_ebs_volume_type: standard
  get_params:
    action: destroy

ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params
  AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
  AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}}
  AWS_DEFAULT_REGION: {{aws-region}}
  BUCKET_PATH: {{tf-bucket-path}}
  BUCKET_NAME: {{tf-bucket-name}}

debug_sleep_anchor: &debug_sleep
  do:
  - task: debug_sleep
    tags: ["gpdb5_ccp_external_worker"]
    config:
      platform: linux
      image_resource:
        type: docker-image
        source:
          repository: alpine
          tag: latest
      run:
        path: 'sh'
        args: ['-c', 'sleep 6h']
    ensure:
      <<: *ccp_destroy

pulse_properties_anchor: &pulse_properties
      PULSE_URL: {{pulse_url}}
      PULSE_USERNAME: {{pulse_username}}
      PULSE_PASSWORD: {{pulse_password}}

## ======================================================================
## jobs
## ======================================================================

# Stage 1: Build and C Unit Tests

jobs:

- name: validate_pipeline
  plan:
  - aggregate:
    - get: gpdb_src
      trigger: true
    - get: centos-gpdb-dev-6
  - task: validate_pipeline
    file: gpdb_src/concourse/tasks/validate_pipeline.yml
    image: centos-gpdb-dev-6

################# Compile Group
- name: gate_compile_start
  plan:
  - aggregate:
    - get: gpdb_src
{'fraction_non_alphanumeric': 0.165847563515202, 'fraction_numerical': 0.017131750659447453, 'mean_word_length': 3.4779311202287704, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7566677', 'n_tokens_mistral': 13774, 'n_tokens_neox': 13211, 'n_words': 1971}