<commit_before>---
# This pipeline requires 7 variables to be set in the CI/CD Settings
# hint for creating the base64 encoded content: `cat .ssh/id_rsa|base64|pbcopy`

# AWS_CREDENTIALS - base64 encoded credentials file (think ~/.aws/credentials)
# AWS_CONFIG - base64 encoded config file (think ~/.aws/config)
# GKE_CONFIG - base64 encoded json file (think ~/.config/gcloud/someconfig.json)
# REGISTRY_PASSWORD - robot token for kraken_lib_rw
# REGISTRY_PASSWORD_K2 - robot token for k2_rw
image: docker:latest

services:
  - docker:dind

variables:
  DOCKER_HOST: tcp://localhost:2375
  DOCKER_DRIVER: overlay2
  REGISTRY_USER: samsung_cnct
  IMAGE_NAME: kraken-lib
  REGISTRY: quay.io
  ROBOT_ACCOUNT: kraken_lib_rw
  ROBOT_ACCOUNT_K2: k2_rw
  KUBERNETES_E2E_TEST_VERSION: v1.9.0
  KUBERNETES_E2E_TEST_VERSION_MINUS_ONE: v1.8.5
  KUBERNETES_E2E_TEST_VERSION_MINUS_TWO: v1.7.11
  ANSIBLE_LINT: "quay.io/samsung_cnct/ansible-lint:latest"
  KRAKEN_TOOLS: "quay.io/samsung_cnct/kraken-tools:latest"
  # this is used to build tagged versions of kraken-lib
  KRAKEN_TOOLS_TAGGED: "quay.io/samsung_cnct/kraken-tools:v0.9"
  E2E_TESTER: "quay.io/samsung_cnct/e2etester:0.2"

stages:
  - vet
  - config-generation
  - pre-test
  - test
  - test-e2e
  - cleanup
  - build
  - publish

#This is the *Default* before_script; some jobs override, and therefore must reference it through YAML pointers.
before_script: &setup_credentials
  - mkdir -p /root/.aws /root/.config/gcloud
  - echo "$AWS_CREDENTIALS" | base64 -d > /root/.aws/credentials
  - echo "$AWS_CONFIG" | base64 -d > /root/.aws/config
  - echo "$GKE_CONFIG" | base64 -d > /root/.config/gcloud/patrickRobot.json
  - mkdir -p /root/.ssh
  - cp -v ${CI_PROJECT_DIR}/.ssh/id_rsa* /root/.ssh/

# This is the *Default* before_script; some jobs override, and therefore must reference it through YAML pointers.
before_script: &setup_credentials
  - mkdir -p /root/.aws /root/.config/gcloud
  - echo "$AWS_CREDENTIALS" | base64 -d > /root/.aws/credentials
  - echo "$AWS_CONFIG" | base64 -d > /root/.aws/config
  - echo "$GKE_CONFIG" | base64 -d > /root/.config/gcloud/patrickRobot.json
  - mkdir -p /root/.ssh
  - cp -v ${CI_PROJECT_DIR}/.ssh/id_rsa* /root/.ssh/

vet:ansible-lint:
  stage: vet
  image: $ANSIBLE_LINT
  before_script: # override the default above
    - echo "Linting doesn't require secrets, huzzah!"
  script:
    - ls ansible/*.yaml ansible/**/*.yaml | xargs -I {} ansible-lint {}


# NOTE: artifacts cannot be pulled from /root/
config-generation:create-ssh-keys:
  stage: config-generation
  image: $KRAKEN_TOOLS
  before_script: # override the default above
    - echo "Generating SSH keys doesn't require secrets, but maybe it should? We could use a CI passphrase for stronger security."
  script:
    - mkdir -p /root/.ssh
    - ./hack/key_gen
    - test -f /root/.ssh/id_rsa || exit $?
    - mkdir -p ${CI_PROJECT_DIR}/.ssh/
    - cp /root/.ssh/id_rsa* ${CI_PROJECT_DIR}/.ssh/
  artifacts:
    paths: 
      - ${CI_PROJECT_DIR}/.ssh/
    expire_in: 14 day

config-generation:create-aws-config-default:
  stage: config-generation
  only:
    - tags
    - branches
  image: $KRAKEN_TOOLS
  artifacts:
    paths:
      - $CI_PROJECT_DIR/cluster/aws-$KUBERNETES_E2E_TEST_VERSION/config.yaml
    expire_in: 14 day
    when: always
  before_script:
    - echo "We don't need secrets to generate config."
  script:
    - hack/gen-test-config.sh aws $KUBERNETES_E2E_TEST_VERSION

config-generation:create-aws-config-minus-one:
  stage: config-generation
  only:
    - "/master/"
    - "/.*test-all.*/"
  image: $KRAKEN_TOOLS
  artifacts:
    paths:
      - $CI_PROJECT_DIR/cluster/aws-$KUBERNETES_E2E_TEST_VERSION_MINUS_ONE/config.yaml
    expire_in: 14 day
  before_script:
    - echo "We don't need secrets to generate config."
  script:
    - hack/gen-test-config.sh aws $KUBERNETES_E2E_TEST_VERSION_MINUS_ONE

config-generation:create-aws-config-minus-two:
  stage: config-generation
  only:
    - "/master/"
    - "/.*test-all.*/"
  image: $KRAKEN_TOOLS
  artifacts:
    paths:
      - $CI_PROJECT_DIR/cluster/aws-$KUBERNETES_E2E_TEST_VERSION_MINUS_TWO/config.yaml
    expire_in: 14 day
  before_script:
    - echo "We don't need secrets to generate config."
  script:
    - hack/gen-test-config.sh aws $KUBERNETES_E2E_TEST_VERSION_MINUS_TWO

config-generation:create-gke-config:
  stage: config-generation
  only:
    - tags
    - branches
  image: $KRAKEN_TOOLS
  artifacts:
    paths:
      - $CI_PROJECT_DIR/cluster/gke-$KUBERNETES_E2E_TEST_VERSION/config.yaml
    expire_in: 14 day 
    when: always
  before_script:
    - echo "We don't need secrets to generate config."
  script:
    - hack/gen-test-config.sh gke $KUBERNETES_E2E_TEST_VERSION

config-generation:create-gke-config:
  stage: config-generation
  only:
    - tags
    - branches
  image: $KRAKEN_TOOLS
  artifacts:
    paths:
      - $CI_PROJECT_DIR/cluster/gke/config.yaml
    expire_in: 14 day 
  before_script:
    - echo "We don't need secrets to generate config."
  script:
    - hack/gen-test-config.sh gke

# Fail as early as possible if the generated dependencies somehow are missing.
test:validate-config:
  stage: pre-test
  script:
    - echo "When this job fails, something's wrong in your CI config."
    - test -f /root/.ssh/id_rsa || exit $?
    - test -f $CI_PROJECT_DIR/cluster/gke-$KUBERNETES_E2E_TEST_VERSION/config.yaml || exit $?
    - test -f $CI_PROJECT_DIR/cluster/aws-$KUBERNETES_E2E_TEST_VERSION/config.yaml || exit $?
    - test -s /root/.aws/config || exit $?
    - test -s /root/.aws/credentials || exit $?
    - test -s /root/.config/gcloud/patrickRobot.json || exit $?

# Fail as early as possible if the generated dependencies somehow are missing.
test:validate-config:
  stage: pre-test
  script:
    - echo "When this job fails, something's wrong in your CI config."
    - test -f /root/.ssh/id_rsa || exit $?
    - test -f $CI_PROJECT_DIR/cluster/gke/config.yaml || exit $?
    - test -f $CI_PROJECT_DIR/cluster/aws/config.yaml || exit $?
    - test -s /root/.aws/config || exit $?
    - test -s /root/.aws/credentials || exit $?
    - test -s /root/.config/gcloud/patrickRobot.json || exit $?


test:aws-dry-run-default:
  stage: pre-test
  only:
    - tags
    - branches
  variables:
    JOB_BASE_NAME: ${CI_PROJECT_NAME}
    BUILD_ID: $CI_PIPELINE_ID
  image: $KRAKEN_TOOLS<commit_msg>remove double nonsense<commit_after>---
# This pipeline requires 7 variables to be set in the CI/CD Settings
# hint for creating the base64 encoded content: `cat .ssh/id_rsa|base64|pbcopy`

# AWS_CREDENTIALS - base64 encoded credentials file (think ~/.aws/credentials)
# AWS_CONFIG - base64 encoded config file (think ~/.aws/config)
# GKE_CONFIG - base64 encoded json file (think ~/.config/gcloud/someconfig.json)
# REGISTRY_PASSWORD - robot token for <PASSWORD>rw
# REGISTRY_PASSWORD_K2 - robot token for k2_rw
image: docker:latest

services:
  - docker:dind

variables:
  DOCKER_HOST: tcp://localhost:2375
  DOCKER_DRIVER: overlay2
  REGISTRY_USER: samsung_cnct
  IMAGE_NAME: kraken-lib
  REGISTRY: quay.io
  ROBOT_ACCOUNT: kraken_lib_rw
  ROBOT_ACCOUNT_K2: k2_rw
  KUBERNETES_E2E_TEST_VERSION: v1.9.0
  KUBERNETES_E2E_TEST_VERSION_MINUS_ONE: v1.8.5
  KUBERNETES_E2E_TEST_VERSION_MINUS_TWO: v1.7.11
  ANSIBLE_LINT: "quay.io/samsung_cnct/ansible-lint:latest"
  KRAKEN_TOOLS: "quay.io/samsung_cnct/kraken-tools:latest"
  # this is used to build tagged versions of kraken-lib
  KRAKEN_TOOLS_TAGGED: "quay.io/samsung_cnct/kraken-tools:v0.9"
  E2E_TESTER: "quay.io/samsung_cnct/e2etester:0.2"

stages:
  - vet
  - config-generation
  - pre-test
  - test
  - test-e2e
  - cleanup
  - build
  - publish

#This is the *Default* before_script; some jobs override, and therefore must reference it through YAML pointers.
before_script: &setup_credentials
  - mkdir -p /root/.aws /root/.config/gcloud
  - echo "$AWS_CREDENTIALS" | base64 -d > /root/.aws/credentials
  - echo "$AWS_CONFIG" | base64 -d > /root/.aws/config
  - echo "$GKE_CONFIG" | base64 -d > /root/.config/gcloud/patrickRobot.json
  - mkdir -p /root/.ssh
  - cp -v ${CI_PROJECT_DIR}/.ssh/id_rsa* /root/.ssh/

vet:ansible-lint:
  stage: vet
  image: $ANSIBLE_LINT
  before_script: # override the default above
    - echo "Linting doesn't require secrets, huzzah!"
  script:
    - ls ansible/*.yaml ansible/**/*.yaml | xargs -I {} ansible-lint {}


# NOTE: artifacts cannot be pulled from /root/
config-generation:create-ssh-keys:
  stage: config-generation
  image: $KRAKEN_TOOLS
  before_script: # override the default above
    - echo "Generating SSH keys doesn't require secrets, but maybe it should? We could use a CI passphrase for stronger security."
  script:
    - mkdir -p /root/.ssh
    - ./hack/key_gen
    - test -f /root/.ssh/id_rsa || exit $?
    - mkdir -p ${CI_PROJECT_DIR}/.ssh/
    - cp /root/.ssh/id_rsa* ${CI_PROJECT_DIR}/.ssh/
  artifacts:
    paths: 
      - ${CI_PROJECT_DIR}/.ssh/
    expire_in: 14 day

config-generation:create-aws-config-default:
  stage: config-generation
  only:
    - tags
    - branches
  image: $KRAKEN_TOOLS
  artifacts:
    paths:
      - $CI_PROJECT_DIR/cluster/aws-$KUBERNETES_E2E_TEST_VERSION/config.yaml
    expire_in: 14 day
    when: always
  before_script:
    - echo "We don't need secrets to generate config."
  script:
    - hack/gen-test-config.sh aws $KUBERNETES_E2E_TEST_VERSION

config-generation:create-aws-config-minus-one:
  stage: config-generation
  only:
    - "/master/"
    - "/.*test-all.*/"
  image: $KRAKEN_TOOLS
  artifacts:
    paths:
      - $CI_PROJECT_DIR/cluster/aws-$KUBERNETES_E2E_TEST_VERSION_MINUS_ONE/config.yaml
    expire_in: 14 day
  before_script:
    - echo "We don't need secrets to generate config."
  script:
    - hack/gen-test-config.sh aws $KUBERNETES_E2E_TEST_VERSION_MINUS_ONE

config-generation:create-aws-config-minus-two:
  stage: config-generation
  only:
    - "/master/"
    - "/.*test-all.*/"
  image: $KRAKEN_TOOLS
  artifacts:
    paths:
      - $CI_PROJECT_DIR/cluster/aws-$KUBERNETES_E2E_TEST_VERSION_MINUS_TWO/config.yaml
    expire_in: 14 day
  before_script:
    - echo "We don't need secrets to generate config."
  script:
    - hack/gen-test-config.sh aws $KUBERNETES_E2E_TEST_VERSION_MINUS_TWO

config-generation:create-gke-config:
  stage: config-generation
  only:
    - tags
    - branches
  image: $KRAKEN_TOOLS
  artifacts:
    paths:
      - $CI_PROJECT_DIR/cluster/gke-$KUBERNETES_E2E_TEST_VERSION/config.yaml
    expire_in: 14 day 
    when: always
  before_script:
    - echo "We don't need secrets to generate config."
  script:
    - hack/gen-test-config.sh gke $KUBERNETES_E2E_TEST_VERSION

# Fail as early as possible if the generated dependencies somehow are missing.
test:validate-config:
  stage: pre-test
  script:
    - echo "When this job fails, something's wrong in your CI config."
    - test -f /root/.ssh/id_rsa || exit $?
    - test -f $CI_PROJECT_DIR/cluster/gke/config.yaml || exit $?
    - test -f $CI_PROJECT_DIR/cluster/aws/config.yaml || exit $?
    - test -s /root/.aws/config || exit $?
    - test -s /root/.aws/credentials || exit $?
    - test -s /root/.config/gcloud/patrickRobot.json || exit $?


test:aws-dry-run-default:
  stage: pre-test
  only:
    - tags
    - branches
  variables:
    JOB_BASE_NAME: ${CI_PROJECT_NAME}
    BUILD_ID: $CI_PIPELINE_ID
  image: $KRAKEN_TOOLS
{'fraction_non_alphanumeric': 0.11675037448233325, 'fraction_numerical': 0.012159661644197727, 'mean_word_length': 3.205261207854761, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7078637', 'n_tokens_mistral': 4440, 'n_tokens_neox': 4261, 'n_words': 934}