<commit_before>#!/bin/bash -x

## =================================================================
## PELOTON PACKAGE INSTALLATION
##
## This script will install all the packages that are needed to
## build and run the DBMS.
##
## Note: On newer versions of Ubuntu (17.04), this script
## will not install the correct version of g++. You will have
## to use 'update-alternatives' to configure the default of
## g++ manually.
##
## Supported environments:
##  * Ubuntu (14.04, 16.04)
##  * Fedora
##  * OSX
## =================================================================


# Determine OS platform
UNAME=$(uname | tr "[:upper:]" "[:lower:]")
# If Linux, try to determine specific distribution
if [ "$UNAME" == "linux" ]; then
    # If available, use LSB to identify distribution
    if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then
        export DISTRO=$(lsb_release -is)
        DISTRO_VER=$(lsb_release -rs)
    # Otherwise, use release info file
    else
        export DISTRO=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1)
        DISTRO_VER=$(cat /etc/*-release | grep "VERSION_ID" | cut -d'=' -f2 | tr -d '"')
    fi
fi
# For everything else (or if above failed), just use generic identifier
[ "$DISTRO" == "" ] && export DISTRO=$UNAME
unset UNAME
DISTRO=$(echo $DISTRO | tr "[:lower:]" "[:upper:]")
TMPDIR=/tmp
TF_VERSION="1.4.0"
TF_TYPE="cpu"


function install_protobuf3.4.0() {
 # Install Relevant tooling
 # Remove any old versions of protobuf
 DISTRIB=$1 # ubuntu/fedora
 if [ "$DISTRIB" == "ubuntu" ]
 then
    sudo apt-get --yes --force-yes remove --purge libprotobuf-dev protobuf-compiler
 elif [ "$DISTRIB" == "fedora" ] 
 then
    sudo dnf -q remove -y protobuf protobuf-devel protobuf-compiler
 else 
    echo "Only Ubuntu and Fedora is supported currently!"
    return 0
 fi
 wget -O protobuf-cpp-3.4.0.tar.gz https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
 tar -xzf protobuf-cpp-3.4.0.tar.gz
 cd protobuf-3.4.0
 ./autogen.sh && ./configure && make -j4 && sudo make install && sudo ldconfig
 cd ..
 # Cleanup
 rm -rf protobuf-3.4.0 protobuf-cpp-3.4.0.tar.gz
}

# Utility function for installing tensorflow components of python/C++
function install_tf() {
 TFCApiFile=$1
 TFBinaryURL=$2
 LinkerConfigCmd=$3
 TARGET_DIRECTORY="/usr/local"
 # Install Tensorflow Python Binary
 sudo -E pip3 install --upgrade ${TFBinaryURL}

 # Install C-API
 TFCApiURL="https://storage.googleapis.com/tensorflow/libtensorflow/${TFCApiFile}"
 wget -O $TFCApiFile $TFCApiURL
 sudo tar -C $TARGET_DIRECTORY -xzf $TFCApiFile || true
 # Configure the Linker
 eval $LinkerConfigCmd
 # Cleanup
 rm -rf ${TFCApiFile}
}

## ------------------------------------------------
## UBUNTU
## ------------------------------------------------
if [ "$DISTRO" = "UBUNTU" ]; then
    MAJOR_VER=$(echo "$DISTRO_VER" | cut -d '.' -f 1)

    # Fix for LLVM-3.7 on Ubuntu 14 + 17
    if [ "$MAJOR_VER" == "14" -o "$MAJOR_VER" == "17" ]; then
        if [ "$MAJOR_VER" == "14" ]; then
            LLVM_PKG_URL="http://llvm.org/apt/trusty/"
            LLVM_PKG_TARGET="llvm-toolchain-trusty-3.7 main"
        fi
        if [ "$MAJOR_VER" == "17" ]; then
            LLVM_PKG_URL="http://apt.llvm.org/artful/"
            LLVM_PKG_TARGET="llvm-toolchain-artful main"
        fi

        if ! grep -q "deb $LLVM_PKG_URL $LLVM_PKG_TARGET" /etc/apt/sources.list; then
            echo -e "\n# Added by Peloton 'packages.sh' script on $(date)\ndeb $LLVM_PKG_URL $LLVM_PKG_TARGET" | sudo tee -a /etc/apt/sources.list > /dev/null
        fi
        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys <KEY>
        sudo apt-get update -qq
        CMAKE_NAME="cmake3"
        FORCE_Y="--force-yes"
    else
        CMAKE_NAME="cmake"
        FORCE_Y=""
    fi

    FORCE_Y=""
    PKG_CMAKE="cmake"
    PKG_LLVM="llvm-3.7"
    PKG_CLANG="clang-3.7"

    # Fix for cmake name change on Ubuntu 14.x and 16.x plus --force-yes deprecation
    if [ "$MAJOR_VER" == "14" ]; then
        PKG_CMAKE="cmake3"
        FORCE_Y="--force-yes"
        TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp34-cp34m-linux_x86_64.whl"
    fi
    if [ "$MAJOR_VER" == "16" ]; then
        TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
    fi
    # Fix for llvm on Ubuntu 17.x
    if [ "$MAJOR_VER" == "17" ]; then
        PKG_LLVM="llvm-3.9"
        PKG_CLANG="clang-3.8"
        TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
    fi
    TFCApiFile="libtensorflow-${TF_TYPE}-linux-x86_64-${TF_VERSION}.tar.gz"
    LinkerConfigCmd="sudo ldconfig"
    sudo apt-get -qq $FORCE_Y --ignore-missing -y install \
        $PKG_CMAKE \
        $PKG_LLVM \<commit_msg>Fix for #1209, packages.sh failures on Ubuntu Xenial<commit_after>#!/bin/bash -ex

## =================================================================
## PELOTON PACKAGE INSTALLATION
##
## This script will install all the packages that are needed to
## build and run the DBMS.
##
## Note: On newer versions of Ubuntu (17.04), this script
## will not install the correct version of g++. You will have
## to use 'update-alternatives' to configure the default of
## g++ manually.
##
## Supported environments:
##  * Ubuntu (14.04, 16.04)
##  * Fedora
##  * OSX
## =================================================================


# Determine OS platform
UNAME=$(uname | tr "[:upper:]" "[:lower:]")
# If Linux, try to determine specific distribution
if [ "$UNAME" == "linux" ]; then
    # If available, use LSB to identify distribution
    if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then
        export DISTRO=$(lsb_release -is)
        DISTRO_VER=$(lsb_release -rs)
    # Otherwise, use release info file
    else
        export DISTRO=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1)
        DISTRO_VER=$(cat /etc/*-release | grep "VERSION_ID" | cut -d'=' -f2 | tr -d '"')
    fi
fi
# For everything else (or if above failed), just use generic identifier
[ "$DISTRO" == "" ] && export DISTRO=$UNAME
unset UNAME
DISTRO=$(echo $DISTRO | tr "[:lower:]" "[:upper:]")
TMPDIR=/tmp
TF_VERSION="1.4.0"
TF_TYPE="cpu"


function install_protobuf3.4.0() {
 # Install Relevant tooling
 # Remove any old versions of protobuf
 DISTRIB=$1 # ubuntu/fedora
 if [ "$DISTRIB" == "ubuntu" ]
 then
    sudo apt-get --yes --force-yes remove --purge libprotobuf-dev protobuf-compiler
 elif [ "$DISTRIB" == "fedora" ] 
 then
    sudo dnf -q remove -y protobuf protobuf-devel protobuf-compiler
 else 
    echo "Only Ubuntu and Fedora is supported currently!"
    return 0
 fi
 wget -O protobuf-cpp-3.4.0.tar.gz https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
 tar -xzf protobuf-cpp-3.4.0.tar.gz
 cd protobuf-3.4.0
 ./autogen.sh && ./configure && make -j4 && sudo make install && sudo ldconfig
 cd ..
 # Cleanup
 rm -rf protobuf-3.4.0 protobuf-cpp-3.4.0.tar.gz
}

# Utility function for installing tensorflow components of python/C++
function install_tf() {
 TFCApiFile=$1
 TFBinaryURL=$2
 LinkerConfigCmd=$3
 TARGET_DIRECTORY="/usr/local"
 # Install Tensorflow Python Binary
 sudo -E pip3 install --upgrade ${TFBinaryURL}

 # Install C-API
 TFCApiURL="https://storage.googleapis.com/tensorflow/libtensorflow/${TFCApiFile}"
 wget -O $TFCApiFile $TFCApiURL
 sudo tar -C $TARGET_DIRECTORY -xzf $TFCApiFile || true
 # Configure the Linker
 eval $LinkerConfigCmd
 # Cleanup
 rm -rf ${TFCApiFile}
}

## ------------------------------------------------
## UBUNTU
## ------------------------------------------------
if [ "$DISTRO" = "UBUNTU" ]; then
    MAJOR_VER=$(echo "$DISTRO_VER" | cut -d '.' -f 1)

    # Fix for LLVM-3.7 on Ubuntu 14 + 17
    if [ "$MAJOR_VER" == "14" -o "$MAJOR_VER" == "17" ]; then
        if [ "$MAJOR_VER" == "14" ]; then
            LLVM_PKG_URL="http://llvm.org/apt/trusty/"
            LLVM_PKG_TARGET="llvm-toolchain-trusty-3.7 main"
        fi
        if [ "$MAJOR_VER" == "17" ]; then
            LLVM_PKG_URL="http://apt.llvm.org/artful/"
            LLVM_PKG_TARGET="llvm-toolchain-artful main"
        fi

        if ! grep -q "deb $LLVM_PKG_URL $LLVM_PKG_TARGET" /etc/apt/sources.list; then
            echo -e "\n# Added by Peloton 'packages.sh' script on $(date)\ndeb $LLVM_PKG_URL $LLVM_PKG_TARGET" | sudo tee -a /etc/apt/sources.list > /dev/null
        fi
        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys <KEY>
        CMAKE_NAME="cmake3"
        FORCE_Y="--force-yes"
    else
        CMAKE_NAME="cmake"
        FORCE_Y=""
    fi

    sudo apt-get update -qq
    FORCE_Y=""
    PKG_CMAKE="cmake"
    PKG_LLVM="llvm-3.7"
    PKG_CLANG="clang-3.7"

    # Fix for cmake name change on Ubuntu 14.x and 16.x plus --force-yes deprecation
    if [ "$MAJOR_VER" == "14" ]; then
        PKG_CMAKE="cmake3"
        FORCE_Y="--force-yes"
        TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp34-cp34m-linux_x86_64.whl"
    fi
    if [ "$MAJOR_VER" == "16" ]; then
        TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
    fi
    # Fix for llvm on Ubuntu 17.x
    if [ "$MAJOR_VER" == "17" ]; then
        PKG_LLVM="llvm-3.9"
        PKG_CLANG="clang-3.8"
        TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
    fi
    TFCApiFile="libtensorflow-${TF_TYPE}-linux-x86_64-${TF_VERSION}.tar.gz"
    LinkerConfigCmd="sudo ldconfig"
    sudo apt-get -qq $FORCE_Y --ignore-missing -y install \
        $PKG_CMAKE \
        $PKG_LLVM \
{'fraction_non_alphanumeric': 0.1937929640113223, 'fraction_numerical': 0.024059846340477153, 'mean_word_length': 2.803537101114956, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5301601', 'n_tokens_mistral': 3754, 'n_tokens_neox': 3474, 'n_words': 833}