<commit_before>#!/bin/bash
# -*- sh-basic-offset: 4; sh-indentation: 4 -*-
# Bootstrap an R/travis environment.

set -e
# Comment out this line for quieter output:
# Or ratherm set it for a lot noisier output
# set -x

CRAN=${CRAN:-"https://cloud.r-project.org"}
BIOC=${BIOC:-"https://bioconductor.org/biocLite.R"}
BIOC_USE_DEVEL=${BIOC_USE_DEVEL:-"TRUE"}
OS=$(uname -s)

## Default version: Now use 4.0, and 3.5 should still work
R_VERSION=${R_VERSION:-"4.0"}

## Possible drat repos, unset by default
DRAT_REPOS=${DRAT_REPOS:-""}

PANDOC_VERSION='1.13.1'
PANDOC_DIR="${HOME}/opt/pandoc"
PANDOC_URL="https://s3.amazonaws.com/rstudio-buildtools/pandoc-${PANDOC_VERSION}.zip"

# MacTeX installs in a new $PATH entry, and there's no way to force
# the *parent* shell to source it from here. So we just manually add
# all the entries to a location we already know to be on $PATH.
#
# TODO(craigcitro): Remove this once we can add `/usr/texbin` to the
# root path.
PATH="${PATH}:/usr/texbin"

R_BUILD_ARGS=${R_BUILD_ARGS-"--no-build-vignettes --no-manual"}
R_CHECK_ARGS=${R_CHECK_ARGS-"--no-vignettes --no-manual --as-cran"}
R_CHECK_INSTALL_ARGS=${R_CHECK_INSTALL_ARGS-"--install-args=--install-tests"}
_R_CHECK_TESTS_NLINES_=0

R_USE_BIOC_CMDS="source('${BIOC}');"\
" tryCatch(useDevel(${BIOC_USE_DEVEL}),"\
" error=function(e) {if (!grepl('already in use', e$message)) {e}});"\
" options(repos=biocinstallRepos());"

Bootstrap() {
    if [[ "Darwin" == "${OS}" ]]; then
        BootstrapMac
    elif [[ "Linux" == "${OS}" ]]; then
        BootstrapLinux
    else
        echo "Unknown OS: ${OS}"
        exit 1
    fi

    if [[ "4.0" == "${R_VERSION}" ]]; then
        echo "Using R 4.0.*"
    elif [[ "3.5" == "${R_VERSION}" ]]; then
        echo "Using R 3.5.*"
    elif [[ "3.4" == "${R_VERSION}" ]]; then
        echo "Using R 3.4 is no longer supported."
        exit 1
    else
        echo "Unknown R_VERSION: ${R_VERSION}"
        exit 1
    fi

    if ! (test -e .Rbuildignore && grep -q 'travis-tool' .Rbuildignore); then
        echo '^travis-tool\.sh$' >>.Rbuildignore
    fi
    if ! (test -e .Rbuildignore && grep -q 'run.sh' .Rbuildignore); then
        echo '^run\.sh$' >> .Rbuildignore
    fi
    if ! (test -e .Rbuildignore && grep -q 'travis_wait' .Rbuildignore); then
        echo '^travis_wait_.*\.log$' >> .Rbuildignore
    fi

    SetRepos
}

SetRepos() {
    echo "local({" >> ~/.Rprofile
    echo "   r <- getOption(\"repos\");" >> ~/.Rprofile
    echo "   r[\"CRAN\"] <- \"${CRAN}\"" >> ~/.Rprofile
    for d in ${DRAT_REPOS}; do
        echo "   r[\"${d}\"] <- \"https://${d}.github.io/drat\"" >> ~/.Rprofile
    done
    echo "   options(repos=r)" >> ~/.Rprofile
    echo "})" >> ~/.Rprofile
}

InstallPandoc() {
    local os_path="$1"
    mkdir -p "${PANDOC_DIR}"
    curl -o /tmp/pandoc-${PANDOC_VERSION}.zip ${PANDOC_URL}
    unzip -j /tmp/pandoc-${PANDOC_VERSION}.zip "pandoc-${PANDOC_VERSION}/${os_path}/pandoc" -d "${PANDOC_DIR}"
    chmod +x "${PANDOC_DIR}/pandoc"
    sudo ln -s "${PANDOC_DIR}/pandoc" /usr/local/bin
    unzip -j /tmp/pandoc-${PANDOC_VERSION}.zip "pandoc-${PANDOC_VERSION}/${os_path}/pandoc-citeproc" -d "${PANDOC_DIR}"
    chmod +x "${PANDOC_DIR}/pandoc-citeproc"
    sudo ln -s "${PANDOC_DIR}/pandoc-citeproc" /usr/local/bin
}

BootstrapLinux() {
    ## Set up our CRAN mirror.
    ## Get the key
    sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys <KEY>
    ## Add the repo
    if [[ "${R_VERSION}" == "4.0" ]]; then
        ## need pinning to ensure repo sorts higher
        echo "Package: *" | sudo tee /etc/apt/preferences.d/c2d4u-pin
        echo "Pin: release o=LP-PPA-c2d4u.team-c2d4u4.0+" | sudo tee -a /etc/apt/preferences.d/c2d4u-pin
        echo "Pin-Priority: 750" | sudo tee -a /etc/apt/preferences.d/c2d4u-pin
        ## now add repo (and update index)
        sudo add-apt-repository "deb ${CRAN}/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
    elif [[ "${R_VERSION}" == "3.5" ]]; then
        sudo add-apt-repository "deb ${CRAN}/bin/linux/ubuntu $(lsb_release -cs)-cran35/"
    elif [[ "${R_VERSION}" == "3.4" ]]; then
        echo "Using R 3.4 is no longer supported."
        exit 1
    fi

    # Add marutter's c2d4u repository.
    if [[ "${R_VERSION}" == "4.0" ]]; then
        sudo add-apt-repository -y "ppa:marutter/rrutter4.0"
        sudo add-apt-repository -y "ppa:c2d4u.team/c2d4u4.0+"
    elif [[ "${R_VERSION}" == "3.5" ]]; then
        sudo add-apt-repository -y "ppa:marutter/rrutter3.5"
        sudo add-apt-repository -y "ppa:marutter/c2d4u3.5"
    elif [[ "${R_VERSION}" == "3.4" ]]; then
        exit "R 3.4 is too old."
        exit 1
        #sudo add-apt-repository -y "ppa:marutter/rrutter"
        #sudo add-apt-repository -y "ppa:marutter/c2d4u"
    fi

    # Update after adding all repositories.  Retry several times to work around
    # flaky connection to Launchpad PPAs.
    Retry sudo apt-get update -qq

    # Install an R development environment. qpdf is also needed for
    # --as-cran checks:
    #   https://stat.ethz.ch/pipermail/r-help//2012-September/335676.html
    # May 2020: we also need devscripts for checkbashism
    Retry sudo apt-get install -y --no-install-recommends r-base-dev r-recommended qpdf devscripts

    # Change permissions for /usr/local/lib/R/site-library
    # This should really be via 'staff adduser travis staff'
    # but that may affect only the next shell
    sudo chmod 2777 /usr/local/lib/R /usr/local/lib/R/site-library

    # Process options
    BootstrapLinuxOptions

    # Report version
    Rscript -e 'sessionInfo()'
}

BootstrapLinuxOptions() {
    if [[ -n "$BOOTSTRAP_LATEX" ]]; then
        # We add a backports PPA for more recent TeX packages.
        sudo add-apt-repository -y "ppa:texlive-backports/ppa"

        Retry sudo apt-get install -y --no-install-recommends \
            texlive-base texlive-latex-base texlive-generic-recommended \
            texlive-fonts-recommended texlive-fonts-extra \
            texlive-extra-utils texlive-latex-recommended texlive-latex-extra \
            texinfo lmodern
    fi
    if [[ -n "$BOOTSTRAP_PANDOC" ]]; then
        InstallPandoc 'linux/debian/x86_64'
    fi
}

BootstrapMac() {
    # Install from latest CRAN binary build for OS X
    wget ${CRAN}/bin/macosx/R-latest.pkg  -O /tmp/R-latest.pkg
<commit_msg>install littler and scripts plus bspm (as opt-in)<commit_after>#!/bin/bash
# -*- sh-basic-offset: 4; sh-indentation: 4 -*-
# Bootstrap an R/travis environment.

set -e
# Comment out this line for quieter output:
# Or ratherm set it for a lot noisier output
# set -x

CRAN=${CRAN:-"https://cloud.r-project.org"}
BIOC=${BIOC:-"https://bioconductor.org/biocLite.R"}
BIOC_USE_DEVEL=${BIOC_USE_DEVEL:-"TRUE"}
OS=$(uname -s)

## Default version: Now use 4.0, and 3.5 should still work
R_VERSION=${R_VERSION:-"4.0"}

## Possible drat repos, unset by default
DRAT_REPOS=${DRAT_REPOS:-""}

PANDOC_VERSION='1.13.1'
PANDOC_DIR="${HOME}/opt/pandoc"
PANDOC_URL="https://s3.amazonaws.com/rstudio-buildtools/pandoc-${PANDOC_VERSION}.zip"

# MacTeX installs in a new $PATH entry, and there's no way to force
# the *parent* shell to source it from here. So we just manually add
# all the entries to a location we already know to be on $PATH.
#
# TODO(craigcitro): Remove this once we can add `/usr/texbin` to the
# root path.
PATH="${PATH}:/usr/texbin"

R_BUILD_ARGS=${R_BUILD_ARGS-"--no-build-vignettes --no-manual"}
R_CHECK_ARGS=${R_CHECK_ARGS-"--no-vignettes --no-manual --as-cran"}
R_CHECK_INSTALL_ARGS=${R_CHECK_INSTALL_ARGS-"--install-args=--install-tests"}
_R_CHECK_TESTS_NLINES_=0

R_USE_BIOC_CMDS="source('${BIOC}');"\
" tryCatch(useDevel(${BIOC_USE_DEVEL}),"\
" error=function(e) {if (!grepl('already in use', e$message)) {e}});"\
" options(repos=biocinstallRepos());"

Bootstrap() {
    if [[ "Darwin" == "${OS}" ]]; then
        BootstrapMac
    elif [[ "Linux" == "${OS}" ]]; then
        BootstrapLinux
    else
        echo "Unknown OS: ${OS}"
        exit 1
    fi

    if [[ "4.0" == "${R_VERSION}" ]]; then
        echo "Using R 4.0.*"
    elif [[ "3.5" == "${R_VERSION}" ]]; then
        echo "Using R 3.5.*"
    elif [[ "3.4" == "${R_VERSION}" ]]; then
        echo "Using R 3.4 is no longer supported."
        exit 1
    else
        echo "Unknown R_VERSION: ${R_VERSION}"
        exit 1
    fi

    if ! (test -e .Rbuildignore && grep -q 'travis-tool' .Rbuildignore); then
        echo '^travis-tool\.sh$' >>.Rbuildignore
    fi
    if ! (test -e .Rbuildignore && grep -q 'run.sh' .Rbuildignore); then
        echo '^run\.sh$' >> .Rbuildignore
    fi
    if ! (test -e .Rbuildignore && grep -q 'travis_wait' .Rbuildignore); then
        echo '^travis_wait_.*\.log$' >> .Rbuildignore
    fi

    SetRepos
}

SetRepos() {
    echo "local({" >> ~/.Rprofile
    echo "   r <- getOption(\"repos\");" >> ~/.Rprofile
    echo "   r[\"CRAN\"] <- \"${CRAN}\"" >> ~/.Rprofile
    for d in ${DRAT_REPOS}; do
        echo "   r[\"${d}\"] <- \"https://${d}.github.io/drat\"" >> ~/.Rprofile
    done
    echo "   options(repos=r)" >> ~/.Rprofile
    echo "})" >> ~/.Rprofile
}

InstallPandoc() {
    local os_path="$1"
    mkdir -p "${PANDOC_DIR}"
    curl -o /tmp/pandoc-${PANDOC_VERSION}.zip ${PANDOC_URL}
    unzip -j /tmp/pandoc-${PANDOC_VERSION}.zip "pandoc-${PANDOC_VERSION}/${os_path}/pandoc" -d "${PANDOC_DIR}"
    chmod +x "${PANDOC_DIR}/pandoc"
    sudo ln -s "${PANDOC_DIR}/pandoc" /usr/local/bin
    unzip -j /tmp/pandoc-${PANDOC_VERSION}.zip "pandoc-${PANDOC_VERSION}/${os_path}/pandoc-citeproc" -d "${PANDOC_DIR}"
    chmod +x "${PANDOC_DIR}/pandoc-citeproc"
    sudo ln -s "${PANDOC_DIR}/pandoc-citeproc" /usr/local/bin
}

BootstrapLinux() {
    ## Set up our CRAN mirror.
    ## Get the key
    sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys <KEY>
    ## Add the repo
    if [[ "${R_VERSION}" == "4.0" ]]; then
        ## need pinning to ensure repo sorts higher
        echo "Package: *" | sudo tee /etc/apt/preferences.d/c2d4u-pin
        echo "Pin: release o=LP-PPA-c2d4u.team-c2d4u4.0+" | sudo tee -a /etc/apt/preferences.d/c2d4u-pin
        echo "Pin-Priority: 750" | sudo tee -a /etc/apt/preferences.d/c2d4u-pin
        ## now add repo (and update index)
        sudo add-apt-repository "deb ${CRAN}/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
    elif [[ "${R_VERSION}" == "3.5" ]]; then
        sudo add-apt-repository "deb ${CRAN}/bin/linux/ubuntu $(lsb_release -cs)-cran35/"
    elif [[ "${R_VERSION}" == "3.4" ]]; then
        echo "Using R 3.4 is no longer supported."
        exit 1
    fi

    # Add marutter's c2d4u repository.
    if [[ "${R_VERSION}" == "4.0" ]]; then
        sudo add-apt-repository -y "ppa:marutter/rrutter4.0"
        sudo add-apt-repository -y "ppa:c2d4u.team/c2d4u4.0+"
    elif [[ "${R_VERSION}" == "3.5" ]]; then
        sudo add-apt-repository -y "ppa:marutter/rrutter3.5"
        sudo add-apt-repository -y "ppa:marutter/c2d4u3.5"
    elif [[ "${R_VERSION}" == "3.4" ]]; then
        exit "R 3.4 is too old."
        exit 1
        #sudo add-apt-repository -y "ppa:marutter/rrutter"
        #sudo add-apt-repository -y "ppa:marutter/c2d4u"
    fi

    # Update after adding all repositories.  Retry several times to work around
    # flaky connection to Launchpad PPAs.
    Retry sudo apt-get update -qq

    # Install an R development environment. qpdf is also needed for
    # --as-cran checks:
    #   https://stat.ethz.ch/pipermail/r-help//2012-September/335676.html
    # May 2020: we also need devscripts for checkbashism
    # Sep 2020: add bspm, littler, docopt
    Retry sudo apt-get install -y --no-install-recommends r-base-dev r-recommended qpdf devscripts r-cran-bspm r-cran-littler r-cran-docopt

    cp -ax /usr/lib/R/site-library/littler/examples/{build.r,check.r,install*.r,update.r} /usr/local/bin
    ## for now also from littler from GH
    install.r remotes
    installGithub.r eddelbuettel/littler
    cp -ax /usr/local/lib/R/site-library/littler/examples/{check.r,install*.r} /usr/local/bin

    # Default to no recommends
    echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/90local-no-recommends

    # Change permissions for /usr/local/lib/R/site-library
    # This should really be via 'staff adduser travis staff'
    # but that may affect only the next shell
    sudo chmod 2777 /usr/local/lib/R /usr/local/lib/R/site-library

    # Process options
    BootstrapLinuxOptions

    # Report version
    Rscript -e 'sessionInfo()'
}

BootstrapLinuxOptions() {
    if [[ -n "$BOOTSTRAP_LATEX" ]]; then
        # We add a backports PPA for more recent TeX packages.
        sudo add-apt-repository -y "ppa:texlive-backports/ppa"

        Retry sudo apt-get install -y --no-install-recommends \
            texlive-base texlive-latex-base texlive-generic-recommended \
            texlive-fonts-recommended texlive-fonts-extra \
            texlive-extra-utils texlive-latex-recommended texlive-latex-extra \
            texinfo lmodern
    fi
    if [[ -n "$BOOTSTRAP_PANDOC" ]]; then
        InstallPandoc 'linux/debian/x86_64'
    fi
    if [[ -n "${USE_BSPM}" ]]; then
        echo "bspm::enable()" >> /etc/R/Rprofile.site
    fi

}

BootstrapMac() {
    # Install from latest CRAN binary build for OS X
    wget ${CRAN}/bin/macosx/R-latest.pkg  -O /tmp/R-latest.pkg

{'fraction_non_alphanumeric': 0.16307206733298263, 'fraction_numerical': 0.01653265198767566, 'mean_word_length': 2.7817561807331628, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 44, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1123632', 'n_tokens_mistral': 5205, 'n_tokens_neox': 4892, 'n_words': 1181}