<commit_before>language: python

python:
  - "2.7"
  - "3.4"

jdk:
  - openjdk7

sudo: false

install:
  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
      wget http://repo.continuum.io/miniconda/Miniconda-3.7.3-Linux-x86_64.sh -O miniconda.sh;
    else
      wget http://repo.continuum.io/miniconda/Miniconda3-3.7.3-Linux-x86_64.sh -O miniconda.sh;
    fi
  - bash miniconda.sh -b -p $HOME/miniconda
  - export PATH="$HOME/miniconda/bin:$PATH"
  - hash -r
  - conda config --set always_yes yes --set changeps1 no
  - conda update -q conda
  - conda info -a
  - conda create --yes -q -n test python=$TRAVIS_PYTHON_VERSION pip numpy scipy scikit-learn scikit-image matplotlib pytest boto
  - source activate test
  - pip install .
  - pip install -r requirements-dev.txt
  - wget http://d3kbcqa49mib13.cloudfront.net/spark-1.5.2-bin-hadoop2.6.tgz
  - tar -xzf spark-1.5.2-bin-hadoop2.6.tgz

script:
  - export SPARK_HOME=`pwd`/spark-1.5.2-bin-hadoop2.6
  - export PYTHONPATH=$SPARK_HOME/python:$PYTHONPATH
  - export PYTHONPATH=$SPARK_HOME/python/lib/py4j-0.8.2.1-src.zip:$PYTHONPATH
  - py.test
  - py.test --engine=spark
<commit_msg>update spark version on travis to 1.6<commit_after>language: python

python:
  - "2.7"
  - "3.4"

jdk:
  - openjdk7

sudo: false

install:
  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
      wget http://repo.continuum.io/miniconda/Miniconda-3.7.3-Linux-x86_64.sh -O miniconda.sh;
    else
      wget http://repo.continuum.io/miniconda/Miniconda3-3.7.3-Linux-x86_64.sh -O miniconda.sh;
    fi
  - bash miniconda.sh -b -p $HOME/miniconda
  - export PATH="$HOME/miniconda/bin:$PATH"
  - hash -r
  - conda config --set always_yes yes --set changeps1 no
  - conda update -q conda
  - conda info -a
  - conda create --yes -q -n test python=$TRAVIS_PYTHON_VERSION pip numpy scipy scikit-learn scikit-image matplotlib pytest boto
  - source activate test
  - pip install .
  - pip install -r requirements-dev.txt
  - wget http://d3kbcqa49mib13.cloudfront.net/spark-1.6.0-bin-hadoop2.6.tgz
  - tar -xzf spark-1.6.0-bin-hadoop2.6.tgz

script:
  - export SPARK_HOME=`pwd`/spark-1.6.0-bin-hadoop2.6
  - export PYTHONPATH=$SPARK_HOME/python:$PYTHONPATH
  - export PYTHONPATH=$SPARK_HOME/python/lib/py4j-0.8.2.1-src.zip:$PYTHONPATH
  - py.test
  - py.test --engine=spark

{'fraction_non_alphanumeric': 0.14539930555555555, 'fraction_numerical': 0.042534722222222224, 'mean_word_length': 2.8545150501672243, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6662232', 'n_tokens_mistral': 1027, 'n_tokens_neox': 983, 'n_words': 180}