<commit_before>#!/bin/sh

set -e

# check to see if install folder exists
if [ ! -d "$HOME/install/bin" ]; then
  # build bmi
  cd $HOME && git clone git://git.mcs.anl.gov/bmi bmi;
  cd bmi && ./prepare && ./configure --enable-shared --enable-bmi-only --prefix=$HOME/install && make && make install;
  # build mpi
  cd $HOME && wget http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz;
  tar -xzf mpich-3.1.4.tar.gz;
  cd mpich-3.1.4 && ./configure --disable-fortran --prefix=$HOME/install && make && make install;
  # build cci
  cd $HOME && wget http://cci-forum.com/wp-content/uploads/2015/07/cci-0.2.0.tar.gz;
  tar -xzf cci-0.2.0.tar.gz;
  cd cci-0.2.0 && ./configure --prefix=$HOME/install && make && make install;
else
  echo "Using cached directory";
fi
<commit_msg>Use cma in CCI sm in travis build<commit_after>#!/bin/sh

set -e

# check to see if install folder exists
if [ ! -d "$HOME/install/bin" ]; then
  # build bmi
  cd $HOME && git clone git://git.mcs.anl.gov/bmi bmi;
  cd bmi && ./prepare && ./configure --enable-shared --enable-bmi-only --prefix=$HOME/install && make && make install;
  # build mpi
  cd $HOME && wget http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz;
  tar -xzf mpich-3.1.4.tar.gz;
  cd mpich-3.1.4 && ./configure --disable-fortran --prefix=$HOME/install && make && make install;
  # build cci
  cd $HOME && wget http://cci-forum.com/wp-content/uploads/2015/07/cci-0.2.0.tar.gz;
  tar -xzf cci-0.2.0.tar.gz;
  cd cci-0.2.0 && ./configure --with-sm-cma --prefix=$HOME/install && make && make install;
else
  echo "Using cached directory";
fi

{'fraction_non_alphanumeric': 0.19201995012468828, 'fraction_numerical': 0.03366583541147132, 'mean_word_length': 2.812351543942993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2450363', 'n_tokens_mistral': 654, 'n_tokens_neox': 631, 'n_words': 150}