<commit_before>sudo: required
dist: trusty
language: cpp

before_script:
  - echo $TRAVIS_OS_NAME
  - |
    if [ "$TRAVIS_OS_NAME" = "osx" ]; then
      brew update
      brew install cmake ninja eigen devil libarchive
      brew link libarchive --force
      export PATH="/usr/local/opt/ccache/libexec:$PATH"
      export CCACHE_CPP2=true
    fi
  - |
    if [ "$TRAVIS_OS_NAME" = "linux" ]; then
      sudo add-apt-repository ppa:nschloe/eigen-backports -y
      sudo add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty main" -y
      sudo add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-4.0 main" -y
      sudo add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-5.0 main" -y
      sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
      sudo apt-get update -qq
      sudo apt-get install -y cmake cmake-data ninja-build libdevil-dev libeigen3-dev libarchive-dev
      export CCACHE_CPP2=true
    fi
  - mkdir build
  - cd build

matrix:
  include:
    - os: linux
      compiler: g++-4.9
      env:
        - TEST="build (g++-4.9)"
      script:
        - sudo apt-get install -y g++-4.9
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="g++-4.9" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: g++-5
      env:
        - TEST="build (g++-5)"
      script:
        - sudo apt-get install -y g++-5
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="g++-5" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: g++-6
      env:
        - TEST="build (g++-6)"
      script:
        - sudo apt-get install -y g++-6
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="g++-6" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: g++-7
      env:
        - TEST="build (g++-7)"
      script:
        - sudo apt-get install -y g++-7
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="g++-7" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-3.6
      env:
        - TEST="build (clang++-3.6)"
      script:
        - sudo apt-get install -y clang-3.6 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-3.6" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

#    - os: linux
#      compiler: clang++-3.7
#      env:
#        - TEST="build (clang++-3.7)"
#      script:
#        - sudo apt-get install -y clang-3.7 libc++abi-dev libc++-dev
#        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-3.7" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
#        - ninja
#        - ./apps/info --system
#        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-3.8
      env:
        - TEST="build (clang++-3.8)"
      script:
        - sudo apt-get install -y clang-3.8 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-3.8" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-3.9
      env:
        - TEST="build (clang++-3.9)"
      script:
        - sudo apt-get install -y clang-3.9 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-3.9" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-4.0
      env:
        - TEST="build (clang++-4.0)"
      script:
        - sudo apt-get install -y clang-4.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-4.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="build (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="ASAN (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON -DNANO_WITH_ASAN=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="MSAN (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON -DNANO_WITH_MSAN=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="TSAN (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON -DNANO_WITH_TSAN=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="clang-tidy (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 clang-tidy-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON -DNANO_WITH_CLANG_TIDY=ON ..
        - ninja tidy | tee output.txt
        - |
          if [[ -n $(grep "warning: " output.txt) ]] || [[ -n $(grep "error: " output.txt) ]]; then
              echo "Clang-tidy detected the following warning and errors:"
              echo ""
              grep --color -E '^|warning: |error: ' output.txt
              exit -1;
          else
              echo -e "\033[1;32m\xE2\x9C\x93 passed:\033[0m $1";
          fi

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="valgrind (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 clang-tidy-5.0 libc++abi-dev libc++-dev valgrind
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ctest -T memcheck -E "test_task_*"

    - os: osx
      compiler: clang++
      osx_image: xcode7.3
      env:
        - TEST="build (xcode7.3)"
      script:
        - cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: osx
      compiler: clang++
      osx_image: xcode8.3
      env:
        - TEST="build (xcode8.3)"
      script:
        - cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: osx
      compiler: clang++
      osx_image: xcode9.1
      env:
        - TEST="build (xcode9.1)"
      script:
        - cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"
<commit_msg>run valgrind on release (no timeout in travis)<commit_after>sudo: required
dist: trusty
language: cpp

before_script:
  - echo $TRAVIS_OS_NAME
  - |
    if [ "$TRAVIS_OS_NAME" = "osx" ]; then
      brew update
      brew install cmake ninja eigen devil libarchive
      brew link libarchive --force
      export PATH="/usr/local/opt/ccache/libexec:$PATH"
      export CCACHE_CPP2=true
    fi
  - |
    if [ "$TRAVIS_OS_NAME" = "linux" ]; then
      sudo add-apt-repository ppa:nschloe/eigen-backports -y
      sudo add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty main" -y
      sudo add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-4.0 main" -y
      sudo add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-5.0 main" -y
      sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
      sudo apt-get update -qq
      sudo apt-get install -y cmake cmake-data ninja-build libdevil-dev libeigen3-dev libarchive-dev
      export CCACHE_CPP2=true
    fi
  - mkdir build
  - cd build

matrix:
  include:
    - os: linux
      compiler: g++-4.9
      env:
        - TEST="build (g++-4.9)"
      script:
        - sudo apt-get install -y g++-4.9
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="g++-4.9" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: g++-5
      env:
        - TEST="build (g++-5)"
      script:
        - sudo apt-get install -y g++-5
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="g++-5" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: g++-6
      env:
        - TEST="build (g++-6)"
      script:
        - sudo apt-get install -y g++-6
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="g++-6" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: g++-7
      env:
        - TEST="build (g++-7)"
      script:
        - sudo apt-get install -y g++-7
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="g++-7" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-3.6
      env:
        - TEST="build (clang++-3.6)"
      script:
        - sudo apt-get install -y clang-3.6 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-3.6" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

#    - os: linux
#      compiler: clang++-3.7
#      env:
#        - TEST="build (clang++-3.7)"
#      script:
#        - sudo apt-get install -y clang-3.7 libc++abi-dev libc++-dev
#        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-3.7" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
#        - ninja
#        - ./apps/info --system
#        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-3.8
      env:
        - TEST="build (clang++-3.8)"
      script:
        - sudo apt-get install -y clang-3.8 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-3.8" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-3.9
      env:
        - TEST="build (clang++-3.9)"
      script:
        - sudo apt-get install -y clang-3.9 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-3.9" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-4.0
      env:
        - TEST="build (clang++-4.0)"
      script:
        - sudo apt-get install -y clang-4.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-4.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="build (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="ASAN (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON -DNANO_WITH_ASAN=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="MSAN (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON -DNANO_WITH_MSAN=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="TSAN (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON -DNANO_WITH_TSAN=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="clang-tidy (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 clang-tidy-5.0 libc++abi-dev libc++-dev
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON -DNANO_WITH_CLANG_TIDY=ON ..
        - ninja tidy | tee output.txt
        - |
          if [[ -n $(grep "warning: " output.txt) ]] || [[ -n $(grep "error: " output.txt) ]]; then
              echo "Clang-tidy detected the following warning and errors:"
              echo ""
              grep --color -E '^|warning: |error: ' output.txt
              exit -1;
          else
              echo -e "\033[1;32m\xE2\x9C\x93 passed:\033[0m $1";
          fi

    - os: linux
      compiler: clang++-5.0
      env:
        - TEST="valgrind (clang++-5.0)"
      script:
        - sudo apt-get install -y clang-5.0 clang-tidy-5.0 libc++abi-dev libc++-dev valgrind
        - cmake -G "Ninja" -DCMAKE_CXX_COMPILER="clang++-5.0" -DCMAKE_BUILD_TYPE=Release -DNANO_WITH_WERROR=ON -DNANO_WITH_LIBCPP=ON ..
        - ninja
        - ctest -T memcheck -E "test_task_*"

    - os: osx
      compiler: clang++
      osx_image: xcode7.3
      env:
        - TEST="build (xcode7.3)"
      script:
        - cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: osx
      compiler: clang++
      osx_image: xcode8.3
      env:
        - TEST="build (xcode8.3)"
      script:
        - cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

    - os: osx
      compiler: clang++
      osx_image: xcode9.1
      env:
        - TEST="build (xcode9.1)"
      script:
        - cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Debug -DNANO_WITH_WERROR=ON ..
        - ninja
        - ./apps/info --system
        - ctest -j 2 -E "test_task_*"

{'fraction_non_alphanumeric': 0.15156934081519394, 'fraction_numerical': 0.020102361719183573, 'mean_word_length': 1.7962068965517242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2423138', 'n_tokens_mistral': 6637, 'n_tokens_neox': 6243, 'n_words': 1277}