<commit_before>variables:
  CRAN_MIRROR: "https://cloud.r-project.org"
  _R_CHECK_FORCE_SUGGESTS_: "false"
  _R_CHECK_NO_STOP_ON_TEST_ERROR_: "true"

stages:
  - dependencies
  - build
  - test
  - integration
  - deploy

.artifacts-template: &artifacts
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - bus

mirror-packages: # download all recursive dependencies of data.table suggests and integration suggests from inst/tests/tests-DESCRIPTION
  stage: dependencies
  tags:
    - linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
  cache:
    paths:
    - bus/$CI_BUILD_NAME/cran
  variables:
    R_BIN_VERSION: "3.6"
    R_DEVEL_BIN_VERSION: "3.7"
  script:
    - echo 'source("ci.R")' >> .Rprofile
    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
    # mirror R dependencies: source, win.binary
    - Rscript -e 'mirror.packages(dcf.dependencies(c("DESCRIPTION","inst/tests/tests-DESCRIPTION"), "all"), repos=c(Sys.getenv("CRAN_MIRROR"), dcf.repos("inst/tests/tests-DESCRIPTION")), repodir="bus/mirror-packages/cran")'
    - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_BIN_VERSION","R_DEVEL_BIN_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
  <<: *artifacts

build: # build data.table sources as tar.gz archive
  stage: build
  tags:
    - linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  dependencies:
  - mirror-packages
  script:
    - Rscript -e 'install.packages("knitr", repos=file.path("file:",normalizePath("bus/mirror-packages/cran")))'
    - rm -r bus
    - echo "Revision:" $CI_BUILD_REF >> ./DESCRIPTION
    - R CMD build .
    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
    - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib/.
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)'
  <<: *artifacts

.test-copy-src: &copy-src
  - &copy-src
    cp $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) .

.test-move-src: &move-src
  - &move-src
    mkdir -p bus/$CI_BUILD_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME

.test-cleanup-src: &cleanup-src
  - &cleanup-src
    rm $(ls -1t data.table_*.tar.gz | head -n 1)

.test-template: &test
  stage: test
  dependencies:
  - mirror-packages
  - build
  <<: *artifacts

.test-lin-template: &test-lin
  <<: *test
  tags:
    - linux

.test-win-template: &test-win
  <<: *test
  tags:
    - windows
  before_script:
    - export PATH="/c/$R_DIR/bin:/c/Rtools/bin:$PATH"
    - rm -rf /tmp/$R_DIR/library && mkdir -p /tmp/$R_DIR/library
    - export R_LIBS_USER="/tmp/$R_DIR/library"

.test-osx-template: &test-osx
  <<: *test
  tags:
    - macosx

test-rel-lin: #  most comprehensive tests, force all suggests, also integration tests, using gcc -O3
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  variables: # unlike CRAN
    _R_CHECK_CRAN_INCOMING_: "FALSE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
    _R_CHECK_FORCE_SUGGESTS_: "TRUE"
    OPENBLAS_MAIN_FREE: "1"
    TEST_DATA_TABLE_WITH_OTHER_PACKAGES: "TRUE"
  before_script:
    - mkdir -p ~/.R
    - echo 'CFLAGS=-g -O3 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O3 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
  script:
    - Rscript -e 'source("ci.R"); install.packages(dcf.dependencies(c("DESCRIPTION","inst/tests/tests-DESCRIPTION"), which="all"))'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src

test-rel-vanilla-lin: # minimal installation, no suggested deps, no vignettes or manuals, measure memory, using gcc -O0 -fno-openmp
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
  variables:
    TEST_DATA_TABLE_MEMTEST: "TRUE"
  before_script:
    - mkdir -p ~/.R
    - echo 'CFLAGS=-g -O0 -fno-openmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O0 -fno-openmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars<commit_msg>LTO compilation flag, also strict compilation warning check (#3878)<commit_after>variables:
  CRAN_MIRROR: "https://cloud.r-project.org"
  _R_CHECK_FORCE_SUGGESTS_: "false"
  _R_CHECK_NO_STOP_ON_TEST_ERROR_: "true"

stages:
  - dependencies
  - build
  - test
  - integration
  - deploy

.artifacts-template: &artifacts
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - bus

mirror-packages: # download all recursive dependencies of data.table suggests and integration suggests from inst/tests/tests-DESCRIPTION
  stage: dependencies
  tags:
    - linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
  cache:
    paths:
    - bus/$CI_BUILD_NAME/cran
  variables:
    R_BIN_VERSION: "3.6"
    R_DEVEL_BIN_VERSION: "3.7"
  script:
    - echo 'source("ci.R")' >> .Rprofile
    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
    # mirror R dependencies: source, win.binary
    - Rscript -e 'mirror.packages(dcf.dependencies(c("DESCRIPTION","inst/tests/tests-DESCRIPTION"), "all"), repos=c(Sys.getenv("CRAN_MIRROR"), dcf.repos("inst/tests/tests-DESCRIPTION")), repodir="bus/mirror-packages/cran")'
    - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_BIN_VERSION","R_DEVEL_BIN_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
  <<: *artifacts

build: # build data.table sources as tar.gz archive
  stage: build
  tags:
    - linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  dependencies:
  - mirror-packages
  script:
    - Rscript -e 'install.packages("knitr", repos=file.path("file:",normalizePath("bus/mirror-packages/cran")))'
    - rm -r bus
    - echo "Revision:" $CI_BUILD_REF >> ./DESCRIPTION
    - R CMD build .
    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
    - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib/.
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)'
  <<: *artifacts

.test-copy-src: &copy-src
  - &copy-src
    cp $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) .

.test-move-src: &move-src
  - &move-src
    mkdir -p bus/$CI_BUILD_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME

.test-cleanup-src: &cleanup-src
  - &cleanup-src
    rm $(ls -1t data.table_*.tar.gz | head -n 1)

.test-template: &test
  stage: test
  dependencies:
  - mirror-packages
  - build
  <<: *artifacts

.test-lin-template: &test-lin
  <<: *test
  tags:
    - linux

.test-win-template: &test-win
  <<: *test
  tags:
    - windows
  before_script:
    - export PATH="/c/$R_DIR/bin:/c/Rtools/bin:$PATH"
    - rm -rf /tmp/$R_DIR/library && mkdir -p /tmp/$R_DIR/library
    - export R_LIBS_USER="/tmp/$R_DIR/library"

.test-osx-template: &test-osx
  <<: *test
  tags:
    - macosx

test-rel-lin: #  most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  variables: # unlike CRAN
    _R_CHECK_CRAN_INCOMING_: "FALSE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
    _R_CHECK_FORCE_SUGGESTS_: "TRUE"
    OPENBLAS_MAIN_FREE: "1"
    TEST_DATA_TABLE_WITH_OTHER_PACKAGES: "TRUE"
  before_script:
    - mkdir -p ~/.R
    - echo 'CFLAGS=-g -O3 -flto -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O3 -flto -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
  script:
    - Rscript -e 'source("ci.R"); install.packages(dcf.dependencies(c("DESCRIPTION","inst/tests/tests-DESCRIPTION"), which="all"))'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src
    - ! grep "warning:" data.table.Rcheck/00install.out

test-rel-vanilla-lin: # minimal installation, no suggested deps, no vignettes or manuals, measure memory, using gcc -O0 -fno-openmp
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
  variables:
    TEST_DATA_TABLE_MEMTEST: "TRUE"
  before_script:
    - mkdir -p ~/.R
    - echo 'CFLAGS=-g -O0 -fno-openmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O0 -fno-openmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
{'fraction_non_alphanumeric': 0.15377697841726617, 'fraction_numerical': 0.006519784172661871, 'mean_word_length': 3.359137677609015, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6148762', 'n_tokens_mistral': 3483, 'n_tokens_neox': 3378, 'n_words': 641}