<commit_before>env:
  - CABALVER=1.18 GHCVER=7.8.3

before_install:
  - travis_retry sudo add-apt-repository -y ppa:hvr/ghc
  - travis_retry sudo apt-get update
  - travis_retry sudo apt-get install cabal-install-$CABALVER ghc-$GHCVER cpphs
  - export PATH=/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
  - |
     if [ $GHCVER = "head" ] || [ ${GHCVER%.*} = "7.8" ] || [ ${GHCVER%.*} = "7.10" ]; then
       travis_retry sudo apt-get install happy-1.19.4 alex-3.1.3
       export PATH=/opt/alex/3.1.3/bin:/opt/happy/1.19.4/bin:$PATH
     else
       travis_retry sudo apt-get install happy alex
     fi

install:
  - cabal --version
  - echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]"
  - travis_retry cabal update
  - cabal install type-eq-0.5
  - cabal install text-1.1.0.0
                  hashable-1.2.2.0
                  mtl-2.1.3.1
                  aeson-0.7.0.6
                  async-2.0.1.5
                  base64-bytestring-1.0.0.1
                  blaze-markup-0.6.1.0
                  blaze-html-0.7.0.2
                  cereal-0.4.1.1
                  byteable-0.1.1
                  cipher-aes-0.2.10
                  entropy-0.3.6
                  clientsession-0.9.1.1
                  prelude-extras-0.4
                  ghc-paths-0.1.0.9
                  haskell-src-exts-1.15.0.1
                  uniplate-1.6.12
                  tagshare-0.0
                  spoon-0.3.1
                  fay-0.20.1.1
                  fay-text-0.3.0.2
                  fay-jquery-0.6.0.2
                  hastache-0.6.1
                  json-0.9.1
                  lens-4.3.3
                  mime-mail-0.4.8.2
                  MissingH-1.2.1.0
                  extensible-exceptions-0.1.1.4
                  MonadCatchIO-transformers-0.3.1.0
                  lifted-base-0.2.3.6
                  mmorph-1.0.4
                  resourcet-1.1.3.3
                  streaming-commons-0.1.10.0
                  auto-update-0.1.2.1
                  fast-logger-2.3.1
                  monad-loops-0.4.2.1
                  stm-chans-3.0.0.3
                  deepseq-generics-0.1.1.2
                  regex-pcre-builtin-0.94.4.8.8.35
                  highlighting-kate-0.5.14
                  pandoc-types-1.12.4.2
                  tagsoup-0.13.3
                  temporary-1.2.0.3
                  texmath-0.6.6.3
                  yaml-0.8.10.1
                  digest-0.0.1.2
                  zip-archive-0.2.3.7
                  pandoc-1.12.3.3
  - cabal install pqueue-1.2.1
                  regex-tdfa-1.2.0
                  unix-compat-0.4.1.4
                  directory-tree-0.12.0
                  heist-0.13.1.2
                  logict-0.6.0.2
                  mwc-random-0.13.3.2
                  enumerator-0.4.20
                  attoparsec-enumerator-0.3.3
                  zlib-bindings-0.1.1.5
                  zlib-enum-0.2.3.1
                  snap-core-0.9.6.3
                  snap-server-0.9.4.5
                  snap-0.13.2.8
                  snap-blaze-0.2.1.2
                  snaplet-fay-0.3.3.7
                  timezone-series-0.1.4
                  timezone-olson-0.1.6
                  --force-reinstalls
  - cabal install persistent-1.3.1.1
                  persistent-sqlite-1.3.0.5
                  persistent-template-1.3.1.4
<commit_msg>Install persistent at the first stage for travis.<commit_after>env:
  - CABALVER=1.18 GHCVER=7.8.3

before_install:
  - travis_retry sudo add-apt-repository -y ppa:hvr/ghc
  - travis_retry sudo apt-get update
  - travis_retry sudo apt-get install cabal-install-$CABALVER ghc-$GHCVER cpphs
  - export PATH=/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
  - |
     if [ $GHCVER = "head" ] || [ ${GHCVER%.*} = "7.8" ] || [ ${GHCVER%.*} = "7.10" ]; then
       travis_retry sudo apt-get install happy-1.19.4 alex-3.1.3
       export PATH=/opt/alex/3.1.3/bin:/opt/happy/1.19.4/bin:$PATH
     else
       travis_retry sudo apt-get install happy alex
     fi

install:
  - cabal --version
  - echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]"
  - travis_retry cabal update
  - cabal install type-eq-0.5
  - cabal install text-1.1.0.0
                  hashable-1.2.2.0
                  mtl-2.1.3.1
                  aeson-0.7.0.6
                  async-2.0.1.5
                  base64-bytestring-1.0.0.1
                  blaze-markup-0.6.1.0
                  blaze-html-0.7.0.2
                  cereal-0.4.1.1
                  byteable-0.1.1
                  cipher-aes-0.2.10
                  entropy-0.3.6
                  clientsession-0.9.1.1
                  prelude-extras-0.4
                  ghc-paths-0.1.0.9
                  haskell-src-exts-1.15.0.1
                  uniplate-1.6.12
                  tagshare-0.0
                  spoon-0.3.1
                  fay-0.20.1.1
                  fay-text-0.3.0.2
                  fay-jquery-0.6.0.2
                  hastache-0.6.1
                  json-0.9.1
                  lens-4.3.3
                  mime-mail-0.4.8.2
                  MissingH-1.2.1.0
                  extensible-exceptions-0.1.1.4
                  MonadCatchIO-transformers-0.3.1.0
                  lifted-base-0.2.3.6
                  mmorph-1.0.4
                  resourcet-1.1.3.3
                  streaming-commons-0.1.10.0
                  auto-update-0.1.2.1
                  fast-logger-2.3.1
                  monad-loops-0.4.2.1
                  stm-chans-3.0.0.3
                  deepseq-generics-0.1.1.2
                  regex-pcre-builtin-0.94.4.8.8.35
                  highlighting-kate-0.5.14
                  pandoc-types-1.12.4.2
                  tagsoup-0.13.3
                  temporary-1.2.0.3
                  texmath-0.6.6.3
                  yaml-0.8.10.1
                  digest-0.0.1.2
                  zip-archive-0.2.3.7
                  pandoc-1.12.3.3
                  persistent-1.3.1.1
                  persistent-sqlite-1.3.0.5
                  persistent-template-1.3.1.4
  - cabal install pqueue-1.2.1
                  regex-tdfa-1.2.0
                  unix-compat-0.4.1.4
                  directory-tree-0.12.0
                  heist-0.13.1.2
                  logict-0.6.0.2
                  mwc-random-0.13.3.2
                  enumerator-0.4.20
                  attoparsec-enumerator-0.3.3
                  zlib-bindings-0.1.1.5
                  zlib-enum-0.2.3.1
                  snap-core-0.9.6.3
                  snap-server-0.9.4.5
                  snap-0.13.2.8
                  snap-blaze-0.2.1.2
                  snaplet-fay-0.3.3.7
                  timezone-series-0.1.4
                  timezone-olson-0.1.6
                  --force-reinstalls

{'fraction_non_alphanumeric': 0.12923167500735944, 'fraction_numerical': 0.08890197232852518, 'mean_word_length': 0.870355078447564, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 96, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1938684', 'n_tokens_mistral': 2672, 'n_tokens_neox': 2477, 'n_words': 283}