<commit_before>language: c++

addons:
  apt:
    sources:
    - ubuntu-toolchain-r-test
    - llvm-toolchain-precise-3.6
    - llvm-toolchain-precise-3.7
    - llvm-toolchain-precise-3.8
    packages:
    - g++-5
    - gcc-5
    - g++-6
    - gcc-6
    - clang-3.6
    - libclang-3.6-dev
    - clang-3.7
    - libclang-3.7-dev
    - clang-3.8
    - libclang-3.8-dev
    - libgtest-dev
    - build-essential
    - checkinstall
    - libreadline-gplv2-dev 
    - libncursesw5-dev 
    - libssl-dev 
    - libsqlite3-dev 
    - tk-dev 
    - libgdbm-dev 
    - libc6-dev 
    - libbz2-dev
    - python-pip

env:
  # Workaround for https://github.com/travis-ci/travis-ci/issues/4681
  matrix:
    - TRAVIS_EMPTY_JOB_WORKAROUND=true

cache:
  directories:
    - ${TRAVIS_BUILD_DIR}/deps/llvm-3.6.2
    - ${TRAVIS_BUILD_DIR}/deps/llvm-3.7.1
    - ${TRAVIS_BUILD_DIR}/deps/llvm-3.8.0

matrix:
  exclude:
    - env: TRAVIS_EMPTY_JOB_WORKAROUND=true
  include:
    # LINUX
    # GCC 5
    - os: linux
      env: COMPILER=g++-5
      compiler: gcc

    # GCC 6
    - os: linux
      env: COMPILER=g++-6
      compiler: gcc
      
install:
  ############################################################################
  # All the dependencies are installed in ${TRAVIS_BUILD_DIR}/deps/
  ############################################################################
  - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps"
  - mkdir -p ${DEPS_DIR} && cd ${DEPS_DIR}
  
  ############################################################################
  # Setup default versions and override compiler if needed
  ############################################################################
  - if [[ "${COMPILER}" != "" ]]; then export CXX=${COMPILER}; fi
  - if [[ "${LLVM_VERSION}" == "default" ]]; then LLVM_VERSION=3.8.0; fi
  
  ############################################################################
  # Install a recent CMake (unless already installed on OS X)
  ############################################################################
  - |
    if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then
      CMAKE_URL="http://www.cmake.org/files/v3.5/cmake-3.5.2-Linux-x86_64.tar.gz"
      mkdir cmake && travis_retry wget --no-check-certificate --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C cmake
      export PATH=${DEPS_DIR}/cmake/bin:${PATH}
    else
      if ! brew ls --version cmake &>/dev/null; then brew install cmake; fi
    fi
    
  ############################################################################
  # Install Clang, libc++ and libc++abi
  ############################################################################
  - |
    if [[ "${LLVM_VERSION}" != "" ]]; then
      LLVM_DIR=${DEPS_DIR}/llvm-${LLVM_VERSION}
      if [[ -z "$(ls -A ${LLVM_DIR})" ]]; then
        LLVM_URL="http://llvm.org/releases/${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz"
        LIBCXX_URL="http://llvm.org/releases/${LLVM_VERSION}/libcxx-${LLVM_VERSION}.src.tar.xz"
        LIBCXXABI_URL="http://llvm.org/releases/${LLVM_VERSION}/libcxxabi-${LLVM_VERSION}.src.tar.xz"
        CLANG_URL="http://llvm.org/releases/${LLVM_VERSION}/clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-14.04.tar.xz"
        mkdir -p ${LLVM_DIR} ${LLVM_DIR}/build ${LLVM_DIR}/projects/libcxx ${LLVM_DIR}/projects/libcxxabi ${LLVM_DIR}/clang
        travis_retry wget --quiet -O - ${LLVM_URL}      | tar --strip-components=1 -xJ -C ${LLVM_DIR}
        travis_retry wget --quiet -O - ${LIBCXX_URL}    | tar --strip-components=1 -xJ -C ${LLVM_DIR}/projects/libcxx
        travis_retry wget --quiet -O - ${LIBCXXABI_URL} | tar --strip-components=1 -xJ -C ${LLVM_DIR}/projects/libcxxabi
        travis_retry wget --quiet -O - ${CLANG_URL}     | tar --strip-components=1 -xJ -C ${LLVM_DIR}/clang
        (cd ${LLVM_DIR}/build && cmake .. -DCMAKE_INSTALL_PREFIX=${LLVM_DIR}/install -DCMAKE_CXX_COMPILER=clang++)
        (cd ${LLVM_DIR}/build/projects/libcxx && make install -j2)
        (cd ${LLVM_DIR}/build/projects/libcxxabi && make install -j2)
      fi
      export CXXFLAGS="-nostdinc++ -isystem ${LLVM_DIR}/install/include/c++/v1"
      export LDFLAGS="-L ${LLVM_DIR}/install/lib -l c++ -l c++abi"
      export LDFLAGS="-L ${LLVM_DIR}/install/lib -l c++ -l c++abi"
      export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${LLVM_DIR}/install/lib"
      export PATH="${LLVM_DIR}/clang/bin:${PATH}"
    fi    
    
  - sudo pip install clang
  - cd /usr/src/gtest
  - sudo cmake CMakeLists.txt
  - sudo make 
  # copy or symlink libgtest.a and libgtest_main.a to your /usr/lib folder
  - sudo cp *.a /usr/lib
  - cd $TRAVIS_BUILD_DIR
script:
  - cd tests
  - ./run
  - cd build && rm -rf * && cmake .. -DCMAKE_BUILD_TYPE=Release && make && ./unit_tests && cd ..
  - cd ..
<commit_msg>[TRAVIS] Add clang-format to packages and run tests with different clang versions<commit_after>language: c++

addons:
  apt:
    sources:
    - ubuntu-toolchain-r-test
    - llvm-toolchain-precise-3.6
    - llvm-toolchain-precise-3.7
    - llvm-toolchain-precise-3.8
    packages:
    - g++-5
    - gcc-5
    - g++-6
    - gcc-6
    - clang-3.6
    - libclang-3.6-dev
    - clang-3.7
    - libclang-3.7-dev
    - clang-3.8
    - libclang-3.8-dev
    - libgtest-dev
    - build-essential
    - checkinstall
    - libreadline-gplv2-dev 
    - libncursesw5-dev 
    - libssl-dev 
    - libsqlite3-dev 
    - tk-dev 
    - libgdbm-dev 
    - libc6-dev 
    - libbz2-dev
    - python-pip
    - clang-format-3.8

env:
  # Workaround for https://github.com/travis-ci/travis-ci/issues/4681
  matrix:
    - TRAVIS_EMPTY_JOB_WORKAROUND=true

cache:
  directories:
    - ${TRAVIS_BUILD_DIR}/deps/llvm-3.6.2
    - ${TRAVIS_BUILD_DIR}/deps/llvm-3.7.1
    - ${TRAVIS_BUILD_DIR}/deps/llvm-3.8.0

matrix:
  exclude:
    - env: TRAVIS_EMPTY_JOB_WORKAROUND=true
  include:
    # Linux
    # GCC 5
    - os: linux
      env: COMPILER=g++-5
      compiler: gcc

    # GCC 6
    - os: linux
      env: COMPILER=g++-6
      compiler: gcc
      
    # Clang 3.6
    - os: linux
      env: COMPILER=clang++-3.6 CCOMPILER=clang-3.6
      compiler: clang
      
    # Clang 3.7
    - os: linux
      env: COMPILER=clang++-3.7 CCOMPILER=clang-3.7
      compiler: clang
      
    # Clang 3.8
    - os: linux
      env: COMPILER=clang++-3.8 CCOMPILER=clang-3.8
      compiler: clang
      
install:
  ############################################################################
  # All the dependencies are installed in ${TRAVIS_BUILD_DIR}/deps/
  ############################################################################
  - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps"
  - mkdir -p ${DEPS_DIR} && cd ${DEPS_DIR}
  
  ############################################################################
  # Setup default versions and override compiler if needed
  ############################################################################
  - if [[ "${COMPILER}" != "" ]]; then export CXX=${COMPILER}; export CC=${CCOMPILER}; fi
  
  ############################################################################
  # Install a recent CMake (unless already installed on OS X)
  ############################################################################
  - |
    if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then
      CMAKE_URL="http://www.cmake.org/files/v3.5/cmake-3.5.2-Linux-x86_64.tar.gz"
      mkdir cmake && travis_retry wget --no-check-certificate --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C cmake
      export PATH=${DEPS_DIR}/cmake/bin:${PATH}
    else
      if ! brew ls --version cmake &>/dev/null; then brew install cmake; fi
    fi
  - sudo pip install clang
  - sudo cp *.a /usr/lib
  - git clone https://github.com/google/googletest.git
  - cd googletest && mkdir build && cd build && cmake .. && make
  - sudo cp -r ../googletest/include/gtest /usr/local/include/
  - sudo cp googlemock/gtest/lib*.a /usr/local/lib
script:
  - cd $TRAVIS_BUILD_DIR
  - cd tests
  - ./run
  - cd build && rm -rf * && cmake .. -DCMAKE_BUILD_TYPE=Release && make && ./unit_tests

{'fraction_non_alphanumeric': 0.26882661996497376, 'fraction_numerical': 0.018263697773329996, 'mean_word_length': 2.798099762470309, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1394350', 'n_tokens_mistral': 2980, 'n_tokens_neox': 2816, 'n_words': 507}