<commit_before>language: java

env:
  global:
    - MAVEN_OPTS="-Xmx512M -XX:+ExitOnOutOfMemoryError"
    - MAVEN_SKIP_CHECKS_AND_DOCS="-Dair.check.skip-all=true -Dmaven.javadoc.skip=true"
    - MAVEN_FAST_INSTALL="-DskipTests $MAVEN_SKIP_CHECKS_AND_DOCS -B -q -T C1"
    - ARTIFACTS_UPLOAD_PATH_BRANCH=travis_build_artifacts/${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}/${TRAVIS_BUILD_NUMBER}
    - ARTIFACTS_UPLOAD_PATH_PR=travis_build_artifacts_pr/${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}/${TRAVIS_BUILD_NUMBER}
    - TEST_FLAGS=""
    - PRODUCT_TESTS_CONFIG=config-empty

matrix:
  include:
    - env: MAVEN_CHECKS=true BUILD_PRESTO_DOCKER=true
    - env: WEBUI_CHECKS=true
    - env: TEST_SPECIFIC_MODULES=presto-tests
    - env: TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P ci-only"
    - env: TEST_SPECIFIC_MODULES=presto-raptor-legacy
    - env: TEST_SPECIFIC_MODULES=presto-accumulo
    - env: TEST_SPECIFIC_MODULES=presto-cassandra
    - env: TEST_SPECIFIC_MODULES=presto-hive
    - env: TEST_SPECIFIC_MODULES=presto-main
    - env: TEST_SPECIFIC_MODULES=presto-orc,presto-parquet
    - env: TEST_SPECIFIC_MODULES=presto-mongodb
    - env: TEST_SPECIFIC_MODULES=presto-redis
    - env: TEST_SPECIFIC_MODULES=presto-sqlserver,presto-postgresql,presto-mysql
    - env: TEST_SPECIFIC_MODULES=presto-phoenix
    - env: TEST_OTHER_MODULES=!presto-tests,!presto-raptor-legacy,!presto-accumulo,!presto-cassandra,!presto-hive,!presto-kudu,!presto-docs,!presto-server,!presto-server-rpm,!presto-main,!presto-orc,!presto-parquet,!presto-mongodb,!presto-redis,!presto-sqlserver,!presto-postgresql,!presto-mysql,!presto-phoenix
    - env: PRODUCT_TESTS_SUITE=suite-1
    - env: PRODUCT_TESTS_SUITE=suite-2
    - env: PRODUCT_TESTS_SUITE=suite-3
    - env: PRODUCT_TESTS_SUITE=suite-4
    - env: HIVE_TESTS=true
    - env: KUDU_TESTS=true

# TODO
#    # HDP 3
#    - if: type IN (push, cron)
#      env: PRODUCT_TESTS_SUITE=suite-1 PRODUCT_TESTS_CONFIG=config-hdp3
#    - if: type IN (push, cron)
#      env: PRODUCT_TESTS_SUITE=suite-2 PRODUCT_TESTS_CONFIG=config-hdp3
#    - if: type IN (push, cron)
#      env: PRODUCT_TESTS_SUITE=suite-3 PRODUCT_TESTS_CONFIG=config-hdp3
#    - if: type IN (push, cron)
#      env: PRODUCT_TESTS_SUITE=suite-4 PRODUCT_TESTS_CONFIG=config-hdp3
#    - if: type IN (push, cron)
#      env: HIVE_TESTS=true PRODUCT_TESTS_CONFIG=config-hdp3

dist: xenial
jdk: openjdk8

cache:
  yarn: true
  directories:
    - $HOME/.m2/repository

git:
  quiet: true

install:
  - travis_retry ./mvnw -v
  - |
    if [[ -v TEST_SPECIFIC_MODULES ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl "${TEST_SPECIFIC_MODULES}" -am
    fi
  - |
    if [[ -v TEST_OTHER_MODULES ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl '!presto-docs,!presto-server,!presto-server-rpm'
    fi
  - |
    if [[ -v PRODUCT_TESTS_SUITE ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl '!presto-docs,!presto-server-rpm'
    fi
  - |
    if [[ -v HIVE_TESTS ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl presto-hive-hadoop2 -am
    fi
  - |
    if [[ -v KUDU_TESTS ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl presto-kudu -am
    fi

before_script:
  - |
    export ARTIFACTS_UPLOAD_PATH=${ARTIFACTS_UPLOAD_PATH_BRANCH}
    if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
      export ARTIFACTS_UPLOAD_PATH="${ARTIFACTS_UPLOAD_PATH_PR}"
    fi

# *** WARNING ***
#
# A single script block must include exactly ONE command.
# Travis runs a script block without the `-e` flag. Any failures in the middle are simply ignored.
#
# Specifying the `-e` flag manually is not recommended.
# It will stop the execution after the very first failure.
# Travis runs the remaining blocks even if one of the scripts returns a non zero status.
# Non zero status is propagated at the end of the build.
script:
  - |
    if [[ -v MAVEN_CHECKS ]]; then
      ./mvnw install -DskipTests -B -T C1 -P ci
    fi
  - |
    if [[ -v BUILD_PRESTO_DOCKER ]]; then
      docker/build-local.sh
    fi
  - |
    if [[ -v WEBUI_CHECKS ]]; then
      presto-main/bin/check_webui.sh
    fi
  - |
    if [[ -v TEST_SPECIFIC_MODULES ]]; then
      ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl "${TEST_SPECIFIC_MODULES}" $TEST_FLAGS
    fi
  - |
    if [[ -v TEST_OTHER_MODULES ]]; then
      ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl "${TEST_OTHER_MODULES}"
    fi
  - |
    if [[ -v PRODUCT_TESTS_SUITE ]]; then
      source "presto-product-tests/conf/product-tests-${PRODUCT_TESTS_CONFIG}.sh"
      "presto-product-tests/bin/product-tests-${PRODUCT_TESTS_SUITE}.sh"
    fi
  - |
    if [[ -v HIVE_TESTS ]]; then
      source "presto-product-tests/conf/product-tests-${PRODUCT_TESTS_CONFIG}.sh"
      presto-hive-hadoop2/bin/run_hive_tests.sh
    fi
  - |
    if [[ -v HIVE_TESTS && -v HIVE_TESTS_AWS_ACCESS_KEY_ID ]]; then
      source "presto-product-tests/conf/product-tests-${PRODUCT_TESTS_CONFIG}.sh"
      env AWS_ACCESS_KEY_ID="${HIVE_TESTS_AWS_ACCESS_KEY_ID}" \
        AWS_SECRET_ACCESS_KEY="${HIVE_TESTS_AWS_SECRET_ACCESS_KEY}" \
        S3_BUCKET_ENDPOINT="${S3_TESTS_BUCKET_ENDPOINT}" \
        S3_BUCKET="${S3_TESTS_BUCKET}" \
        presto-hive-hadoop2/bin/run_hive_s3_tests.sh
    fi
  - |
    if [[ -v HIVE_TESTS && -v HIVE_TESTS_AWS_ACCESS_KEY_ID ]]; then
      env AWS_ACCESS_KEY_ID="${HIVE_TESTS_AWS_ACCESS_KEY_ID}" \
        AWS_SECRET_ACCESS_KEY="${HIVE_TESTS_AWS_SECRET_ACCESS_KEY}" \
        ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl presto-hive -P test-hive-glue
    fi
  - |
    if [[ -v KUDU_TESTS ]]; then
      presto-kudu/bin/run_kudu_tests.sh null &&
      presto-kudu/bin/run_kudu_tests.sh "" &&
      presto-kudu/bin/run_kudu_tests.sh presto:: &&
      true
    fi

before_cache:
  # Make the cache stable between builds by removing build output
  - rm -rf $HOME/.m2/repository/io/prestosql

notifications:
  slack:
    secure: <KEY>

before_deploy:
  - mkdir /tmp/artifacts
  - cp -n presto-server/target/presto-server-*.tar.gz /tmp/artifacts
  - cp -n presto-server-rpm/target/presto-server-rpm-*.x86_64.rpm /tmp/artifacts
  - cp -n presto-product-tests/target/presto-product-tests-*-executable.jar /tmp/artifacts
  - cp -n presto-jdbc/target/presto-jdbc-*.jar /tmp/artifacts
  - cp -n presto-cli/target/presto-cli-*-executable.jar /tmp/artifacts
  - echo $TRAVIS_COMMIT > /tmp/artifacts/git-revision.txt
  - echo "<script>location='https://travis-ci.org/${TRAVIS_REPO_SLUG}/builds/${TRAVIS_BUILD_ID}'</script>"
    > /tmp/artifacts/travis_build.html
  - ls -lah /tmp/artifacts

deploy:
  on:
    all_branches: true
    condition: -v DEPLOY_S3_ACCESS_KEY && -v MAVEN_CHECKS
  provider: s3
  access_key_id: ${DEPLOY_S3_ACCESS_KEY}
  secret_access_key: ${DEPLOY_S3_SECRET_KEY}
  bucket: ${DEPLOY_S3_BUCKET}
  skip_cleanup: true
  local-dir: /tmp/artifacts
  upload-dir: ${ARTIFACTS_UPLOAD_PATH}
  acl: public_read

after_script:
- |
  if [[ -v DEPLOY_S3_ACCESS_KEY ]]; then
    sudo pip install awscli
    export AWS_ACCESS_KEY_ID="${DEPLOY_S3_ACCESS_KEY}"
    export AWS_SECRET_ACCESS_KEY="${DEPLOY_S3_SECRET_KEY}"

    JOB_ARTIFACTS_URL_PREFIX="s3://${DEPLOY_S3_BUCKET}/${ARTIFACTS_UPLOAD_PATH}/travis_jobs/${TRAVIS_JOB_NUMBER}-run"
    JOB_RUN_ATTEMPTS=$( aws s3 ls "${JOB_ARTIFACTS_URL_PREFIX}" | wc -l | tr -d '[:space:]' )
    JOB_STATUS=$( [ "${TRAVIS_TEST_RESULT}" == "0" ] && echo SUCCESS || echo FAILURE )

    mkdir -p /tmp/job_artifacts/
    rsync -av -m \
      --include='**/' \
      --include='**/surefire-reports/**.xml' \
      --include='**/surefire-reports/emailable-report.html' \
      --include='**/product-tests-presto-jvm-error-file.log' \
      --include='**/test-reports/junitreports/**.xml' \
      --include='**/test-reports/emailable-report.html' \
      --exclude='*' \
      . /tmp/job_artifacts/
    wget "https://api.travis-ci.org/jobs/${TRAVIS_JOB_ID}/log.txt?deansi=true" -O /tmp/job_artifacts/log.txt

    aws s3 sync /tmp/job_artifacts "${JOB_ARTIFACTS_URL_PREFIX}_$((JOB_RUN_ATTEMPTS + 1))-${JOB_STATUS}" --quiet
  fi
<commit_msg>Propagate error when reading configuration fails<commit_after>language: java

env:
  global:
    - MAVEN_OPTS="-Xmx512M -XX:+ExitOnOutOfMemoryError"
    - MAVEN_SKIP_CHECKS_AND_DOCS="-Dair.check.skip-all=true -Dmaven.javadoc.skip=true"
    - MAVEN_FAST_INSTALL="-DskipTests $MAVEN_SKIP_CHECKS_AND_DOCS -B -q -T C1"
    - ARTIFACTS_UPLOAD_PATH_BRANCH=travis_build_artifacts/${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}/${TRAVIS_BUILD_NUMBER}
    - ARTIFACTS_UPLOAD_PATH_PR=travis_build_artifacts_pr/${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}/${TRAVIS_BUILD_NUMBER}
    - TEST_FLAGS=""
    - PRODUCT_TESTS_CONFIG=config-empty

matrix:
  include:
    - env: MAVEN_CHECKS=true BUILD_PRESTO_DOCKER=true
    - env: WEBUI_CHECKS=true
    - env: TEST_SPECIFIC_MODULES=presto-tests
    - env: TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P ci-only"
    - env: TEST_SPECIFIC_MODULES=presto-raptor-legacy
    - env: TEST_SPECIFIC_MODULES=presto-accumulo
    - env: TEST_SPECIFIC_MODULES=presto-cassandra
    - env: TEST_SPECIFIC_MODULES=presto-hive
    - env: TEST_SPECIFIC_MODULES=presto-main
    - env: TEST_SPECIFIC_MODULES=presto-orc,presto-parquet
    - env: TEST_SPECIFIC_MODULES=presto-mongodb
    - env: TEST_SPECIFIC_MODULES=presto-redis
    - env: TEST_SPECIFIC_MODULES=presto-sqlserver,presto-postgresql,presto-mysql
    - env: TEST_SPECIFIC_MODULES=presto-phoenix
    - env: TEST_OTHER_MODULES=!presto-tests,!presto-raptor-legacy,!presto-accumulo,!presto-cassandra,!presto-hive,!presto-kudu,!presto-docs,!presto-server,!presto-server-rpm,!presto-main,!presto-orc,!presto-parquet,!presto-mongodb,!presto-redis,!presto-sqlserver,!presto-postgresql,!presto-mysql,!presto-phoenix
    - env: PRODUCT_TESTS_SUITE=suite-1
    - env: PRODUCT_TESTS_SUITE=suite-2
    - env: PRODUCT_TESTS_SUITE=suite-3
    - env: PRODUCT_TESTS_SUITE=suite-4
    - env: HIVE_TESTS=true
    - env: KUDU_TESTS=true

# TODO
#    # HDP 3
#    - if: type IN (push, cron)
#      env: PRODUCT_TESTS_SUITE=suite-1 PRODUCT_TESTS_CONFIG=config-hdp3
#    - if: type IN (push, cron)
#      env: PRODUCT_TESTS_SUITE=suite-2 PRODUCT_TESTS_CONFIG=config-hdp3
#    - if: type IN (push, cron)
#      env: PRODUCT_TESTS_SUITE=suite-3 PRODUCT_TESTS_CONFIG=config-hdp3
#    - if: type IN (push, cron)
#      env: PRODUCT_TESTS_SUITE=suite-4 PRODUCT_TESTS_CONFIG=config-hdp3
#    - if: type IN (push, cron)
#      env: HIVE_TESTS=true PRODUCT_TESTS_CONFIG=config-hdp3

dist: xenial
jdk: openjdk8

cache:
  yarn: true
  directories:
    - $HOME/.m2/repository

git:
  quiet: true

install:
  - travis_retry ./mvnw -v
  - |
    if [[ -v TEST_SPECIFIC_MODULES ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl "${TEST_SPECIFIC_MODULES}" -am
    fi
  - |
    if [[ -v TEST_OTHER_MODULES ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl '!presto-docs,!presto-server,!presto-server-rpm'
    fi
  - |
    if [[ -v PRODUCT_TESTS_SUITE ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl '!presto-docs,!presto-server-rpm'
    fi
  - |
    if [[ -v HIVE_TESTS ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl presto-hive-hadoop2 -am
    fi
  - |
    if [[ -v KUDU_TESTS ]]; then
      ./mvnw install $MAVEN_FAST_INSTALL -pl presto-kudu -am
    fi

before_script:
  - |
    export ARTIFACTS_UPLOAD_PATH=${ARTIFACTS_UPLOAD_PATH_BRANCH}
    if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
      export ARTIFACTS_UPLOAD_PATH="${ARTIFACTS_UPLOAD_PATH_PR}"
    fi

# *** WARNING ***
#
# A single script block must include exactly ONE command.
# Travis runs a script block without the `-e` flag. Any failures in the middle are simply ignored.
#
# Specifying the `-e` flag manually is not recommended.
# It will stop the execution after the very first failure.
# Travis runs the remaining blocks even if one of the scripts returns a non zero status.
# Non zero status is propagated at the end of the build.
script:
  - |
    if [[ -v MAVEN_CHECKS ]]; then
      ./mvnw install -DskipTests -B -T C1 -P ci
    fi
  - |
    if [[ -v BUILD_PRESTO_DOCKER ]]; then
      docker/build-local.sh
    fi
  - |
    if [[ -v WEBUI_CHECKS ]]; then
      presto-main/bin/check_webui.sh
    fi
  - |
    if [[ -v TEST_SPECIFIC_MODULES ]]; then
      ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl "${TEST_SPECIFIC_MODULES}" $TEST_FLAGS
    fi
  - |
    if [[ -v TEST_OTHER_MODULES ]]; then
      ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl "${TEST_OTHER_MODULES}"
    fi
  - |
    if [[ -v PRODUCT_TESTS_SUITE ]]; then
      source "presto-product-tests/conf/product-tests-${PRODUCT_TESTS_CONFIG}.sh" &&
      "presto-product-tests/bin/product-tests-${PRODUCT_TESTS_SUITE}.sh"
    fi
  - |
    if [[ -v HIVE_TESTS ]]; then
      source "presto-product-tests/conf/product-tests-${PRODUCT_TESTS_CONFIG}.sh" &&
      presto-hive-hadoop2/bin/run_hive_tests.sh
    fi
  - |
    if [[ -v HIVE_TESTS && -v HIVE_TESTS_AWS_ACCESS_KEY_ID ]]; then
      source "presto-product-tests/conf/product-tests-${PRODUCT_TESTS_CONFIG}.sh" &&
      env AWS_ACCESS_KEY_ID="${HIVE_TESTS_AWS_ACCESS_KEY_ID}" \
        AWS_SECRET_ACCESS_KEY="${HIVE_TESTS_AWS_SECRET_ACCESS_KEY}" \
        S3_BUCKET_ENDPOINT="${S3_TESTS_BUCKET_ENDPOINT}" \
        S3_BUCKET="${S3_TESTS_BUCKET}" \
        presto-hive-hadoop2/bin/run_hive_s3_tests.sh
    fi
  - |
    if [[ -v HIVE_TESTS && -v HIVE_TESTS_AWS_ACCESS_KEY_ID ]]; then
      env AWS_ACCESS_KEY_ID="${HIVE_TESTS_AWS_ACCESS_KEY_ID}" \
        AWS_SECRET_ACCESS_KEY="${HIVE_TESTS_AWS_SECRET_ACCESS_KEY}" \
        ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl presto-hive -P test-hive-glue
    fi
  - |
    if [[ -v KUDU_TESTS ]]; then
      presto-kudu/bin/run_kudu_tests.sh null &&
      presto-kudu/bin/run_kudu_tests.sh "" &&
      presto-kudu/bin/run_kudu_tests.sh presto:: &&
      true
    fi

before_cache:
  # Make the cache stable between builds by removing build output
  - rm -rf $HOME/.m2/repository/io/prestosql

notifications:
  slack:
    secure: <KEY>

before_deploy:
  - mkdir /tmp/artifacts
  - cp -n presto-server/target/presto-server-*.tar.gz /tmp/artifacts
  - cp -n presto-server-rpm/target/presto-server-rpm-*.x86_64.rpm /tmp/artifacts
  - cp -n presto-product-tests/target/presto-product-tests-*-executable.jar /tmp/artifacts
  - cp -n presto-jdbc/target/presto-jdbc-*.jar /tmp/artifacts
  - cp -n presto-cli/target/presto-cli-*-executable.jar /tmp/artifacts
  - echo $TRAVIS_COMMIT > /tmp/artifacts/git-revision.txt
  - echo "<script>location='https://travis-ci.org/${TRAVIS_REPO_SLUG}/builds/${TRAVIS_BUILD_ID}'</script>"
    > /tmp/artifacts/travis_build.html
  - ls -lah /tmp/artifacts

deploy:
  on:
    all_branches: true
    condition: -v DEPLOY_S3_ACCESS_KEY && -v MAVEN_CHECKS
  provider: s3
  access_key_id: ${DEPLOY_S3_ACCESS_KEY}
  secret_access_key: ${DEPLOY_S3_SECRET_KEY}
  bucket: ${DEPLOY_S3_BUCKET}
  skip_cleanup: true
  local-dir: /tmp/artifacts
  upload-dir: ${ARTIFACTS_UPLOAD_PATH}
  acl: public_read

after_script:
- |
  if [[ -v DEPLOY_S3_ACCESS_KEY ]]; then
    sudo pip install awscli
    export AWS_ACCESS_KEY_ID="${DEPLOY_S3_ACCESS_KEY}"
    export AWS_SECRET_ACCESS_KEY="${DEPLOY_S3_SECRET_KEY}"

    JOB_ARTIFACTS_URL_PREFIX="s3://${DEPLOY_S3_BUCKET}/${ARTIFACTS_UPLOAD_PATH}/travis_jobs/${TRAVIS_JOB_NUMBER}-run"
    JOB_RUN_ATTEMPTS=$( aws s3 ls "${JOB_ARTIFACTS_URL_PREFIX}" | wc -l | tr -d '[:space:]' )
    JOB_STATUS=$( [ "${TRAVIS_TEST_RESULT}" == "0" ] && echo SUCCESS || echo FAILURE )

    mkdir -p /tmp/job_artifacts/
    rsync -av -m \
      --include='**/' \
      --include='**/surefire-reports/**.xml' \
      --include='**/surefire-reports/emailable-report.html' \
      --include='**/product-tests-presto-jvm-error-file.log' \
      --include='**/test-reports/junitreports/**.xml' \
      --include='**/test-reports/emailable-report.html' \
      --exclude='*' \
      . /tmp/job_artifacts/
    wget "https://api.travis-ci.org/jobs/${TRAVIS_JOB_ID}/log.txt?deansi=true" -O /tmp/job_artifacts/log.txt

    aws s3 sync /tmp/job_artifacts "${JOB_ARTIFACTS_URL_PREFIX}_$((JOB_RUN_ATTEMPTS + 1))-${JOB_STATUS}" --quiet
  fi

{'fraction_non_alphanumeric': 0.13310878243512975, 'fraction_numerical': 0.005988023952095809, 'mean_word_length': 3.4253381175821143, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5119744', 'n_tokens_mistral': 6693, 'n_tokens_neox': 6694, 'n_words': 1005}