<commit_before>###############################################
#  Basic checks
###############################################

sudo apt-get update
sudo apt-get upgrade
R
update.packages(ask=FALSE)
# See here for getting R.oo to install: https://github.com/HenrikBengtsson/R.utils/issues/29
q("no")

# Ensure latest version of R otherwise problems with CRAN not finding dependents that depend on latest R
# e.g. mirror may have been disabled in sources.list when upgrading ubuntu

# Ensure no non-ASCII, other than in README.md is ok
# tests.Rraw in particular have failed CRAN Solaris (only) due to this.
# No unicode either. Put these tests in DtNonAsciiTests package.
grep -RI --exclude-dir=".git" --exclude="*.md" --exclude="*~" --color='auto' -P -n "[\x80-\xFF]" data.table/
grep -RI --exclude-dir=".git" --exclude="*.md" --exclude="*~" --color='auto' -n "[\]u[0-9]" data.table/

# Ensure no calls to omp_set_num_threads() [to avoid affecting other packages and base R]
grep --exclude="data.table/src/openmp-utils.c" omp_set_num_threads data.table/src/*

# Ensure no calls to omp_get_max_threads() also since access should be via getDTthreads()
grep --exclude="data.table/src/openmp-utils.c" omp_get_max_threads data.table/src/*

# Ensure all #pragama omp parallel directives include a num_threads() clause
grep "pragma omp parallel" data.table/src/*.c

# Ensure all .Call's first argument are unquoted.  TODO - change to use INHERITS()
grep "[.]Call(\"" data.table/R/*.R

# Ensure no Rprintf in init.c
grep "Rprintf" data.table/src/init.c

# workaround for IBM AIX - ensure no globals named 'nearest' or 'class'.
# See https://github.com/Rdatatable/data.table/issues/1351
grep "nearest *=" data.table/src/*.c  # none
grep "class *=" data.table/src/*.c    # quite a few but none global

# No undefined type punning of the form:  *(long long *)&REAL(column)[i]
# Failed clang 3.9.1 -O3 due to this, I think.
grep "&REAL" data.table/src/*.c

# No use of long long, instead use int64_t. TODO
# grep "long long" data.table/src/*.c

# No tabs in C or R code (sorry <NAME>)
grep -P "\t" data.table/R/*.R
grep -P "\t" data.table/src/*.c


# seal leak potential where two unprotected API calls are passed to the same
# function call, usually involving install() or mkChar()
# Greppable thanks to single lines and wide screens
# See comments in init.c
cd data.table/src
grep install.*alloc *.c   --exclude init.c
grep install.*Scalar *.c
grep alloc.*install *.c   --exclude init.c
grep Scalar.*install *.c
grep mkChar.*alloc *.c
grep mkChar.*Scalar *.c
grep alloc.*mkChar *.c
grep Scalar.*mkChar *.c
grep "getAttrib.*mk" *.c   # use sym_* in getAttrib calls
grep "PROTECT *( *getAttrib" *.c  # attributes are already protected
grep "\"starts\"" *.c     --exclude init.c
grep "setAttrib(" *.c      # scan all setAttrib calls manually as a double-check
grep "install(" *.c       --exclude init.c   # TODO: perhaps in future pre-install all constants

# ScalarInteger and ScalarString allocate and must be PROTECTed unless i) returned (which protects),
# or ii) passed to setAttrib (which protects, providing leak-seals above are ok)
# ScalarLogical in R now returns R's global TRUE but from R 3.1.0; Apr 2014. Before that it allocates.
# Aside: ScalarInteger may return globals for small integers in future version of R.
grep ScalarInteger *.c   # Check all Scalar* either PROTECTed, return-ed or passed to setAttrib.
grep ScalarLogical *.c   # When we move R dependency to 3.1.0+, no need to protect ScalarLogical
grep ScalarString *.c

cd
R
cc(clean=TRUE)  # to compile with -pedandic. Also use very latest gcc (currently gcc-7) as CRAN does
q("no")
R CMD build data.table
R CMD check data.table_1.10.1.tar.gz --as-cran
R CMD INSTALL data.table_1.10.1.tar.gz
R
require(data.table)
test.data.table()
test.data.table(verbose=TRUE)  # since main.R no longer tests verbose mode

# Upload to win-builder, both release and dev


###############################################
#  R 3.1.0 (stated dependency)
###############################################

### ONE TIME BUILD
sudo apt-get -y build-dep r-base
cd ~/build
wget http://cran.stat.ucla.edu/src/base/R-3/R-3.1.0.tar.gz
tar xvf R-3.1.0.tar.gz
cd R-3.1.0
./configure --without-recommended-packages
make
alias R310=~/build/R-3.1.0/bin/R
### END ONE TIME BUILD

R310 CMD INSTALL ./data.table_1.10.5.tar.gz
R310
require(data.table)
test.data.table()


###############################################
#  Compiles from source when OpenMP is disabled
###############################################
vi ~/.R/Makevars
# Make line SHLIB_OPENMP_CFLAGS= active to remove -fopenmp
R CMD build .
R CMD INSTALL data.table_1.10.5.tar.gz   # ensure that -fopenmp is missing and there are no warnings
R
require(data.table)   # observe startup message about no OpenMP detected
test.data.table()


###############################################
#  valgrind
###############################################

vi ~/.R/Makevars  # make the -O0 -g line active, for info on source lines with any problems
R --vanilla CMD INSTALL data.table_1.10.5.tar.gz
R -d "valgrind --tool=memcheck --leak-check=full --show-leak-kinds=definite" --vanilla
require(data.table)
require(bit64)
test.data.table()

gctorture(TRUE)   # very very slow, though. Don't run with suggests tests.
test.data.table()

# Investigated and ignore :
# Tests 648 and 1262 (see their comments) have single precision issues under valgrind that don't occur on CRAN, even Solaris.
# Ignore all "set address range perms" warnings :
#   http://stackoverflow.com/questions/13558067/what-does-this-valgrind-warning-mean-warning-set-address-range-perms
# Ignore heap summaries around test 1705 and 1707/1708 due to the fork() test opening/closing, I guess.
# Tests 1729.4, 1729.8, 1729.11, 1729.13 again have precision issues under valgrind only.
# Leaks for tests 1738.5, 1739.3 but no data.table .c lines are flagged, rather libcairo.so
#   and libfontconfig.so via GEMetricInfo and GEStrWidth in libR.so

vi ~/.R/Makevars  # make the -O3 line active again


###############################################
#  R-devel with UBSAN and ASAN on too
###############################################

cd ~/build
wget -N ftp://ftp.stat.math.ethz.ch/pub/Software/R/R-devel.tar.gz
rm -rf R-devel
tar xvf R-devel.tar.gz
cd R-devel
# Following R-exts#4.3.3
# (clang 3.6.0 works but gcc 4.9.2 fails in R's distance.c:256 error: ‘*.Lubsan_data0’ not specified in enclosing parallel)
./configure CC="clang-5.0 -fsanitize=undefined,address -fno-sanitize=float-divide-by-zero -fno-omit-frame-pointer" CFLAGS="-g -Og -Wall -pedantic" --without-recommended-packages --disable-byte-compiled-packages
make
alias Rdevel='~/build/R-devel/bin/R --vanilla'
Rdevel<commit_msg>For R-devel testing under ASAN added --disable-openmp to get past .Lubsan_data0 error because gcc reports ASAN fail line number where clang-5.0 does not<commit_after>###############################################
#  Basic checks
###############################################

sudo apt-get update
sudo apt-get upgrade
R
update.packages(ask=FALSE)
# See here for getting R.oo to install: https://github.com/HenrikBengtsson/R.utils/issues/29
q("no")

# Ensure latest version of R otherwise problems with CRAN not finding dependents that depend on latest R
# e.g. mirror may have been disabled in sources.list when upgrading ubuntu

# Ensure no non-ASCII, other than in README.md is ok
# tests.Rraw in particular have failed CRAN Solaris (only) due to this.
# No unicode either. Put these tests in DtNonAsciiTests package.
grep -RI --exclude-dir=".git" --exclude="*.md" --exclude="*~" --color='auto' -P -n "[\x80-\xFF]" data.table/
grep -RI --exclude-dir=".git" --exclude="*.md" --exclude="*~" --color='auto' -n "[\]u[0-9]" data.table/

# Ensure no calls to omp_set_num_threads() [to avoid affecting other packages and base R]
grep --exclude="data.table/src/openmp-utils.c" omp_set_num_threads data.table/src/*

# Ensure no calls to omp_get_max_threads() also since access should be via getDTthreads()
grep --exclude="data.table/src/openmp-utils.c" omp_get_max_threads data.table/src/*

# Ensure all #pragama omp parallel directives include a num_threads() clause
grep "pragma omp parallel" data.table/src/*.c

# Ensure all .Call's first argument are unquoted.  TODO - change to use INHERITS()
grep "[.]Call(\"" data.table/R/*.R

# Ensure no Rprintf in init.c
grep "Rprintf" data.table/src/init.c

# workaround for IBM AIX - ensure no globals named 'nearest' or 'class'.
# See https://github.com/Rdatatable/data.table/issues/1351
grep "nearest *=" data.table/src/*.c  # none
grep "class *=" data.table/src/*.c    # quite a few but none global

# No undefined type punning of the form:  *(long long *)&REAL(column)[i]
# Failed clang 3.9.1 -O3 due to this, I think.
grep "&REAL" data.table/src/*.c

# No use of long long, instead use int64_t. TODO
# grep "long long" data.table/src/*.c

# No tabs in C or R code (sorry <NAME>)
grep -P "\t" data.table/R/*.R
grep -P "\t" data.table/src/*.c


# seal leak potential where two unprotected API calls are passed to the same
# function call, usually involving install() or mkChar()
# Greppable thanks to single lines and wide screens
# See comments in init.c
cd data.table/src
grep install.*alloc *.c   --exclude init.c
grep install.*Scalar *.c
grep alloc.*install *.c   --exclude init.c
grep Scalar.*install *.c
grep mkChar.*alloc *.c
grep mkChar.*Scalar *.c
grep alloc.*mkChar *.c
grep Scalar.*mkChar *.c
grep "getAttrib.*mk" *.c   # use sym_* in getAttrib calls
grep "PROTECT *( *getAttrib" *.c  # attributes are already protected
grep "\"starts\"" *.c     --exclude init.c
grep "setAttrib(" *.c      # scan all setAttrib calls manually as a double-check
grep "install(" *.c       --exclude init.c   # TODO: perhaps in future pre-install all constants

# ScalarInteger and ScalarString allocate and must be PROTECTed unless i) returned (which protects),
# or ii) passed to setAttrib (which protects, providing leak-seals above are ok)
# ScalarLogical in R now returns R's global TRUE but from R 3.1.0; Apr 2014. Before that it allocates.
# Aside: ScalarInteger may return globals for small integers in future version of R.
grep ScalarInteger *.c   # Check all Scalar* either PROTECTed, return-ed or passed to setAttrib.
grep ScalarLogical *.c   # When we move R dependency to 3.1.0+, no need to protect ScalarLogical
grep ScalarString *.c

cd
R
cc(clean=TRUE)  # to compile with -pedandic. Also use very latest gcc (currently gcc-7) as CRAN does
q("no")
R CMD build data.table
R CMD check data.table_1.10.1.tar.gz --as-cran
R CMD INSTALL data.table_1.10.1.tar.gz
R
require(data.table)
test.data.table()
test.data.table(verbose=TRUE)  # since main.R no longer tests verbose mode

# Upload to win-builder, both release and dev


###############################################
#  R 3.1.0 (stated dependency)
###############################################

### ONE TIME BUILD
sudo apt-get -y build-dep r-base
cd ~/build
wget http://cran.stat.ucla.edu/src/base/R-3/R-3.1.0.tar.gz
tar xvf R-3.1.0.tar.gz
cd R-3.1.0
./configure --without-recommended-packages
make
alias R310=~/build/R-3.1.0/bin/R
### END ONE TIME BUILD

R310 CMD INSTALL ./data.table_1.10.5.tar.gz
R310
require(data.table)
test.data.table()


###############################################
#  Compiles from source when OpenMP is disabled
###############################################
vi ~/.R/Makevars
# Make line SHLIB_OPENMP_CFLAGS= active to remove -fopenmp
R CMD build .
R CMD INSTALL data.table_1.10.5.tar.gz   # ensure that -fopenmp is missing and there are no warnings
R
require(data.table)   # observe startup message about no OpenMP detected
test.data.table()


###############################################
#  valgrind
###############################################

vi ~/.R/Makevars  # make the -O0 -g line active, for info on source lines with any problems
R --vanilla CMD INSTALL data.table_1.10.5.tar.gz
R -d "valgrind --tool=memcheck --leak-check=full --show-leak-kinds=definite" --vanilla
require(data.table)
require(bit64)
test.data.table()

gctorture(TRUE)   # very very slow, though. Don't run with suggests tests.
test.data.table()

# Investigated and ignore :
# Tests 648 and 1262 (see their comments) have single precision issues under valgrind that don't occur on CRAN, even Solaris.
# Ignore all "set address range perms" warnings :
#   http://stackoverflow.com/questions/13558067/what-does-this-valgrind-warning-mean-warning-set-address-range-perms
# Ignore heap summaries around test 1705 and 1707/1708 due to the fork() test opening/closing, I guess.
# Tests 1729.4, 1729.8, 1729.11, 1729.13 again have precision issues under valgrind only.
# Leaks for tests 1738.5, 1739.3 but no data.table .c lines are flagged, rather libcairo.so
#   and libfontconfig.so via GEMetricInfo and GEStrWidth in libR.so

vi ~/.R/Makevars  # make the -O3 line active again


###############################################
#  R-devel with UBSAN and ASAN on too
###############################################

cd ~/build
wget -N ftp://ftp.stat.math.ethz.ch/pub/Software/R/R-devel.tar.gz
rm -rf R-devel
tar xvf R-devel.tar.gz
cd R-devel
# Following R-exts#4.3.3
./configure CC="gcc -fsanitize=undefined,address -fno-sanitize=float-divide-by-zero -fno-omit-frame-pointer" CFLAGS="-g -Og -Wall -pedantic" LIBS="-lpthread" --without-recommended-packages --disable-byte-compiled-packages --disable-openmp
# For ubsan, disabled openmp otherwise gcc fails in R's distance.c:256 error: ‘*.Lubsan_data0’ not specified in enclosing parallel
# UBSAN gives direct line number under gcc but not clang it seems. clang-5.0 has been helpful too, though.
# If use later gcc-8, add F77=gfortran-8
# LIBS="-lpthread" otherwise ld error about DSO missing
make
alias Rdevel='~/build/R-devel/bin/R --vanilla'
Rdevel
{'fraction_non_alphanumeric': 0.1808264701632712, 'fraction_numerical': 0.02174541251264268, 'mean_word_length': 4.0688392530208715, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1725903', 'n_tokens_mistral': 4858, 'n_tokens_neox': 4529, 'n_words': 1562}