<commit_before>version: 2
jobs:

  build:
    environment:
      TZ: "/usr/share/zoneinfo/America/Los_Angeles"
      SCRATCH: "/scratch"
    machine:
      image: circleci/classic:201711-01
    working_directory: /tmp/src/fmriprep
    steps:
      - checkout
      - run:
          name: Check whether build should be skipped
          command: |
            if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
              echo "Only docs build"
              circleci step halt
            fi

      - restore_cache:
          keys:
            - build-v4-{{ .Branch }}-{{ epoch }}
            - build-v4-{{ .Branch }}-
            - build-v4-master-
            - build-v4-
          paths:
            - /tmp/docker
      - run:
          name: Set up Docker registry
          command: |
            docker run -d -p 5000:5000 --restart=always --name=registry \
                -v /tmp/docker:/var/lib/registry registry:2
      - run:
          name: Pull images
          command: |
            set +e
            docker pull localhost:5000/ubuntu
            success=$?
            set -e
            if [[ "$success" = "0" ]]; then
                echo "Pulling from local registry"
                docker tag localhost:5000/ubuntu ubuntu:xenial-20200114
                docker pull localhost:5000/fmriprep
                docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
                docker tag localhost:5000/fmriprep poldracklab/fmriprep
            else
                echo "Pulling from Docker Hub"
                docker pull ubuntu:xenial-20200114
                docker tag ubuntu:xenial-20200114 localhost:5000/ubuntu
                docker push localhost:5000/ubuntu
                docker pull poldracklab/fmriprep:latest
            fi
      - run:
          name: Build Docker image
          no_output_timeout: 60m
          command: |
            export PY3=$(pyenv versions | grep '3\.' |
                         sed -e 's/.* 3\./3./' -e 's/ .*//')
            pyenv local $PY3
            # Get version, update files.
            THISVERSION=$( python3 get_version.py )
            if [[ ${THISVERSION:0:1} == "0" ]] ; then
              echo "WARNING: latest git tag could not be found"
              echo "Please, make sure you fetch all tags from upstream with"
              echo "the command ``git fetch --tags --verbose`` and push"
              echo "them to your fork with ``git push origin --tags``"
            fi
            sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'${CIRCLE_TAG:-$THISVERSION}'/" wrapper/fmriprep_docker.py
            sed -i -E "s/(var version = )'[A-Za-z0-9.-]+'/\1'${CIRCLE_TAG:-$THISVERSION}'/" docs/citing.rst
            sed -i "s/title = {FMRIPrep}/title = {FMRIPrep ${CIRCLE_TAG:-$THISVERSION}}/" fmriprep/data/boilerplate.bib
            # Build docker image
            e=1 && for i in {1..5}; do
              docker build --rm \
                --cache-from=poldracklab/fmriprep \
                -t poldracklab/fmriprep:latest \
                --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
                --build-arg VCS_REF=`git rev-parse --short HEAD` \
                --build-arg VERSION="${CIRCLE_TAG:-$THISVERSION}" . \
              && e=0 && break || sleep 15
            done && [ "$e" -eq "0" ]
      - run:
          name: Docker push to local registry
          no_output_timeout: 40m
          command: |
            docker tag poldracklab/fmriprep:latest localhost:5000/fmriprep
            docker push localhost:5000/fmriprep
      - run:
          name: Docker registry garbage collection
          command: |
            docker exec -it registry /bin/registry garbage-collect --delete-untagged \
              /etc/docker/registry/config.yml

      - persist_to_workspace:
          root: /tmp
          paths:
            - src/fmriprep
      - save_cache:
         key: build-v4-{{ .Branch }}-{{ epoch }}
         paths:
            - /tmp/docker

  get_data:
    machine:
      # Ubuntu 14.04 with Docker 17.10.0-ce
      image: circleci/classic:201711-01
    working_directory: /home/circleci/data
    steps:
      - restore_cache:
          keys:
            - data-v7-{{ .Branch }}-{{ .Revision }}
            - data-v7-{{ .Branch }}
            - data-v7-
      - run:
          name: Get test data from ds000005
          command: |
            mkdir -p /tmp/data
            if [[ ! -d /tmp/data/ds005 ]]; then
              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
                -O ds005_downsampled.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/57f32a429ad5a101f977eb75"
              tar xvzf ds005_downsampled.tar.gz -C /tmp/data/
            else
              echo "Dataset ds000005 was cached"
            fi
      - run:
          name: Get test data from ds000054
          command: |
            if [[ ! -d /tmp/data/ds054 ]]; then
              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
                -O ds054_downsampled.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/57f32c22594d9001ef91bf9e"
              tar xvzf ds054_downsampled.tar.gz -C /tmp/data/
            else
              echo "Dataset ds000054 was cached"
            fi
      - run:
          name: Get test data from ds000210
          command: |
            if [[ ! -d /tmp/data/ds210 ]]; then
              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
                -O ds210_downsampled.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/5ae9e37b9a64d7000ce66c21"
              tar xvzf ds210_downsampled.tar.gz -C /tmp/data/
            else
              echo "Dataset ds000210 was cached"
            fi
      - run:
          name: Get FreeSurfer derivatives for ds000005
          command: |
            if [[ ! -d /tmp/ds005/freesurfer ]]; then
              mkdir -p /tmp/ds005
              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
                -O ds005_derivatives_freesurfer.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/58fe59eb594d900250960180"
              tar xvzf ds005_derivatives_freesurfer.tar.gz -C /tmp/ds005
            else
              echo "FreeSurfer derivatives of ds000005 were cached"
            fi
      - run:
          name: Store FreeSurfer license file
          command: |
            mkdir -p /tmp/fslicense
            cd /tmp/fslicense
            echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IGxpY2Vuc2UudHh0Cg==" | base64 -d | sh
      - run:
          name: Create Nipype config files
          command: |
            mkdir -p /tmp/ds005 /tmp/ds054 /tmp/ds210
            printf "[execution]\nstop_on_first_crash = true\n" > /tmp/ds005/nipype.cfg
            echo "poll_sleep_duration = 0.01" >> /tmp/ds005/nipype.cfg
            echo "hash_method = content" >> /tmp/ds005/nipype.cfg
            cp /tmp/ds005/nipype.cfg /tmp/ds054/nipype.cfg
            cp /tmp/ds005/nipype.cfg /tmp/ds210/nipype.cfg
      - persist_to_workspace:
          root: /tmp
          paths:
            - fslicense
            - ds005/nipype.cfg
            - ds054/nipype.cfg
            - ds210/nipype.cfg
      - save_cache:
         key: data-v7-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
         paths:
            - /tmp/data
            - /tmp/ds005/freesurfer

  test_pytest:
    machine:
      image: circleci/classic:201711-01
    working_directory: /tmp/src/fmriprep
    steps:
      - checkout:
          path: /home/circleci/src/fmriprep
      - run:
          name: Check whether build should be skipped
          command: |
            cd /home/circleci/src/fmriprep
            if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
              echo "Only docs build"
              circleci step halt
            fi
            if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[skip[ _]?tests\]' )" != "" ]]; then
              echo "Skipping pytest job"
              circleci step halt
            fi

      - attach_workspace:
          at: /tmp
      - restore_cache:
          keys:
            - build-v4-{{ .Branch }}-{{ epoch }}
            - build-v4-{{ .Branch }}-
            - build-v4-
          paths:
            - /tmp/docker
      - restore_cache:
          keys:
            - data-v8-{{ .Revision }}
      - run:
          name: Set-up a Docker registry
          command: |
            docker run -d -p 5000:5000 --restart=always --name=registry \
                -v /tmp/docker:/var/lib/registry registry:2
      - run:
          name: Pull images from local registry
          command: |
            docker pull localhost:5000/fmriprep
            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
      - run:
          name: Run fMRIPrep tests
          no_output_timeout: 2h
          command: |
            docker run -ti --rm=false \
              -e TEST_READONLY_FILESYSTEM=1 -v $HOME:/home/readonly:ro \
              --entrypoint="pytest" poldracklab/fmriprep:latest \
              /src/fmriprep/fmriprep -svx --doctest-modules

      - run:
          name: Test fmriprep-wrapper (Python 2)
          command: |
            export PY2=$(pyenv versions | grep '2\.' |
                         sed -e 's/.* 2\./2./' -e 's/ .*//')
            pyenv local $PY2
            echo -n "Python version: "
            python --version
            pip install --upgrade pip setuptools
            pip install --upgrade wrapper/
            which fmriprep-docker
            fmriprep-docker -i poldracklab/fmriprep:latest --help
            fmriprep-docker -i poldracklab/fmriprep:latest --version
      - run:
          name: Test fmriprep-wrapper (Python 3)
          command: |
            export PY3=$(pyenv versions | grep '3\.' |
                         sed -e 's/.* 3\./3./' -e 's/ .*//')
            pyenv local $PY3
            echo -n "Python version: "
            python --version
            pip install --upgrade pip setuptools
            pip install --upgrade wrapper/
            which fmriprep-docker
            fmriprep-docker -i poldracklab/fmriprep:latest --help
            fmriprep-docker -i poldracklab/fmriprep:latest --version
      - store_artifacts:
          path: /tmp/data/reports

  build_docs:
    docker:
      - image: python:3.7.4
    working_directory: /tmp/src/fmriprep
    environment:
      - FSLOUTPUTTYPE: 'NIFTI'
    steps:
      - checkout
      - run:
          name: Check whether build should be skipped
          command: |
            if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[skip[ _]?docs\]' )" != "" ]]; then
              echo "Skipping doc building job"
              circleci step halt
            fi
      - run:
          name: Check Python version and upgrade pip
          command: |
            python --version
            python -m pip install -U pip
      - run:
          name: Install graphviz
          command: |
              apt-get update
              apt-get install -y graphviz
      - run:
          name: Install Requirements (may contain pinned versions)
          command: python -m pip install -r docs/requirements.txt
      - run:
          name: Install fMRIPrep
          command: python -m pip install ".[doc]"
      - run:
          name: Build documentation
          command: make SPHINXOPTS="-W" -C docs html
      - store_artifacts:
          path: /tmp/src/fmriprep/docs/_build/html

  ds005:
    machine:
      image: circleci/classic:201711-01
    working_directory: /tmp/ds005
    environment:
      - FS_LICENSE: /tmp/fslicense/license.txt
    steps:
      - checkout:
          path: /home/circleci/src/fmriprep
      - run:
          name: Check whether build should be skipped
          command: |
            cd /home/circleci/src/fmriprep
            if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
              echo "Only docs build"
              circleci step halt
            fi
            if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[skip[ _]?ds005\]' )" != "" ]]; then
              echo "Skipping ds000005 build"
              circleci step halt
            fi

      - attach_workspace:
          at: /tmp
      - restore_cache:
          keys:
            - build-v4-{{ .Branch }}-{{ epoch }}
            - build-v4-{{ .Branch }}-
            - build-v4-
          paths:
            - /tmp/docker
      - restore_cache:
          keys:
            - data-v7-{{ .Branch }}-{{ .Revision }}
      - restore_cache:
          keys:
            - ds005-anat-v17-{{ .Branch }}-{{ .Revision }}
            - ds005-anat-v17-{{ .Branch }}
            - ds005-anat-v17-master
            - ds005-anat-v17-
      - run:
          name: Set-up a Docker registry
          command: |
            docker run -d -p 5000:5000 --restart=always --name=registry \
                -v /tmp/docker:/var/lib/registry registry:2
      - run:
          name: Pull images from local registry
          command: |
            docker pull localhost:5000/fmriprep
            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
      - run:
          name: Setting up test
          command: |
            mkdir -p /tmp/ds005/derivatives && sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/derivatives && sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/derivatives
            pip install --upgrade pip setuptools
            pip install --upgrade /tmp/src/fmriprep/wrapper/
      - run:
          name: Run anatomical workflow on ds005
          no_output_timeout: 2h
          command: |
            mkdir -p /tmp/ds005/work /tmp/ds005/derivatives
            sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/derivatives && \
                sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/derivatives
            sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/work && \
                sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/work
            fmriprep-docker -i poldracklab/fmriprep:latest \
                -e FMRIPREP_DEV 1 -u $(id -u) \
                --config $PWD/nipype.cfg -w /tmp/ds005/work \
                /tmp/data/ds005 /tmp/ds005/derivatives participant \
                --fs-subjects-dir /tmp/ds005/freesurfer \
                --skull-strip-template OASIS30ANTs:res-1 \
                --output-spaces MNI152NLin2009cAsym MNI152NLin6Asym \
                --sloppy --write-graph --mem_mb 4096 \
                --nthreads 2 --anat-only -vv
      - run:
          name: Clean-up after anatomical run
          command: |
            rm -rf /tmp/ds005/work/fmriprep_wf/fsdir*
            rm -rf /tmp/ds005/work/reportlets
            rm -rf /tmp/ds005/derivatives/fmriprep
      - save_cache:
         key: ds005-anat-v17-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
         paths:
            - /tmp/ds005/work

      - run:
          name: Run full fMRIPrep on ds005 (LegacyMultiProc plugin)
          no_output_timeout: 2h
          command: |
            sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/derivatives && \
                sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/derivatives
            sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/work && \
                sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/work
            fmriprep-docker -i poldracklab/fmriprep:latest \
                -e FMRIPREP_DEV 1 -u $(id -u) \
                --config $PWD/nipype.cfg -w /tmp/ds005/work \
                /tmp/data/ds005 /tmp/ds005/derivatives participant \
                --fs-subjects-dir /tmp/ds005/freesurfer \
                --sloppy --write-graph --use-syn-sdc --mem_mb 4096 \
                --use-aroma \
                --skull-strip-template OASIS30ANTs:res-1 \
                --output-spaces MNI152NLin2009cAsym fsaverage5 fsnative MNI152NLin6Asym anat \
                --use-plugin /home/circleci/src/fmriprep/.circleci/legacy.yml \
                --nthreads 2 --cifti-output -vv
      - run:
          name: Move intermediate results for second run
          command: |
            mkdir -p /tmp/ds005/derivatives_partial
            sudo cp -a /tmp/ds005/work /tmp/ds005/work_partial
            sudo rm -rf /tmp/ds005/work_partial/fmriprep_wf/single_subject_01_wf/func_preproc_task_mixedgamblestask_run_02_wf/ica_aroma_wf
      - run:
          name: Checking outputs of full fMRIPrep run
          command: |
            mkdir -p /tmp/ds005/test
            find /tmp/ds005/derivatives -path */figures -prune -o -name "*" -print | sed s+/tmp/ds005/derivatives/++ | sort > /tmp/ds005/test/outputs.out
            diff /tmp/src/fmriprep/.circleci/ds005_outputs.txt /tmp/ds005/test/outputs.out
            exit $?
      - run:
          name: Re-run fMRIPrep on single run of task data
          no_output_timeout: 2h
          command: |
            rm /tmp/data/ds005/sub-01/func/*_run-01_*
            fmriprep-docker -i poldracklab/fmriprep:latest \
                -e FMRIPREP_DEV 1 -u $(id -u) \
                --config $PWD/nipype.cfg -w /tmp/ds005/work_partial \
                /tmp/data/ds005 /tmp/ds005/derivatives_partial participant \
                --fs-subjects-dir /tmp/ds005/freesurfer \
                --sloppy --write-graph --use-syn-sdc --mem_mb 4096 \
                --output-spaces MNI152NLin2009cAsym fsaverage5 fsnative MNI152NLin6Asym anat \<commit_msg>CI: Test that Docker image can run a common set of output spaces without network access<commit_after>version: 2
jobs:

  build:
    environment:
      TZ: "/usr/share/zoneinfo/America/Los_Angeles"
      SCRATCH: "/scratch"
    machine:
      image: circleci/classic:201711-01
    working_directory: /tmp/src/fmriprep
    steps:
      - checkout
      - run:
          name: Check whether build should be skipped
          command: |
            if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
              echo "Only docs build"
              circleci step halt
            fi

      - restore_cache:
          keys:
            - build-v4-{{ .Branch }}-{{ epoch }}
            - build-v4-{{ .Branch }}-
            - build-v4-master-
            - build-v4-
          paths:
            - /tmp/docker
      - run:
          name: Set up Docker registry
          command: |
            docker run -d -p 5000:5000 --restart=always --name=registry \
                -v /tmp/docker:/var/lib/registry registry:2
      - run:
          name: Pull images
          command: |
            set +e
            docker pull localhost:5000/ubuntu
            success=$?
            set -e
            if [[ "$success" = "0" ]]; then
                echo "Pulling from local registry"
                docker tag localhost:5000/ubuntu ubuntu:xenial-20200114
                docker pull localhost:5000/fmriprep
                docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
                docker tag localhost:5000/fmriprep poldracklab/fmriprep
            else
                echo "Pulling from Docker Hub"
                docker pull ubuntu:xenial-20200114
                docker tag ubuntu:xenial-20200114 localhost:5000/ubuntu
                docker push localhost:5000/ubuntu
                docker pull poldracklab/fmriprep:latest
            fi
      - run:
          name: Build Docker image
          no_output_timeout: 60m
          command: |
            export PY3=$(pyenv versions | grep '3\.' |
                         sed -e 's/.* 3\./3./' -e 's/ .*//')
            pyenv local $PY3
            # Get version, update files.
            THISVERSION=$( python3 get_version.py )
            if [[ ${THISVERSION:0:1} == "0" ]] ; then
              echo "WARNING: latest git tag could not be found"
              echo "Please, make sure you fetch all tags from upstream with"
              echo "the command ``git fetch --tags --verbose`` and push"
              echo "them to your fork with ``git push origin --tags``"
            fi
            sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'${CIRCLE_TAG:-$THISVERSION}'/" wrapper/fmriprep_docker.py
            sed -i -E "s/(var version = )'[A-Za-z0-9.-]+'/\1'${CIRCLE_TAG:-$THISVERSION}'/" docs/citing.rst
            sed -i "s/title = {FMRIPrep}/title = {FMRIPrep ${CIRCLE_TAG:-$THISVERSION}}/" fmriprep/data/boilerplate.bib
            # Build docker image
            e=1 && for i in {1..5}; do
              docker build --rm \
                --cache-from=poldracklab/fmriprep \
                -t poldracklab/fmriprep:latest \
                --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
                --build-arg VCS_REF=`git rev-parse --short HEAD` \
                --build-arg VERSION="${CIRCLE_TAG:-$THISVERSION}" . \
              && e=0 && break || sleep 15
            done && [ "$e" -eq "0" ]
      - run:
          name: Docker push to local registry
          no_output_timeout: 40m
          command: |
            docker tag poldracklab/fmriprep:latest localhost:5000/fmriprep
            docker push localhost:5000/fmriprep
      - run:
          name: Docker registry garbage collection
          command: |
            docker exec -it registry /bin/registry garbage-collect --delete-untagged \
              /etc/docker/registry/config.yml

      - persist_to_workspace:
          root: /tmp
          paths:
            - src/fmriprep
      - save_cache:
         key: build-v4-{{ .Branch }}-{{ epoch }}
         paths:
            - /tmp/docker

  get_data:
    machine:
      # Ubuntu 14.04 with Docker 17.10.0-ce
      image: circleci/classic:201711-01
    working_directory: /home/circleci/data
    steps:
      - restore_cache:
          keys:
            - data-v7-{{ .Branch }}-{{ .Revision }}
            - data-v7-{{ .Branch }}
            - data-v7-
      - run:
          name: Get test data from ds000005
          command: |
            mkdir -p /tmp/data
            if [[ ! -d /tmp/data/ds005 ]]; then
              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
                -O ds005_downsampled.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/57f32a429ad5a101f977eb75"
              tar xvzf ds005_downsampled.tar.gz -C /tmp/data/
            else
              echo "Dataset ds000005 was cached"
            fi
      - run:
          name: Get test data from ds000054
          command: |
            if [[ ! -d /tmp/data/ds054 ]]; then
              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
                -O ds054_downsampled.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/57f32c22594d9001ef91bf9e"
              tar xvzf ds054_downsampled.tar.gz -C /tmp/data/
            else
              echo "Dataset ds000054 was cached"
            fi
      - run:
          name: Get test data from ds000210
          command: |
            if [[ ! -d /tmp/data/ds210 ]]; then
              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
                -O ds210_downsampled.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/5ae9e37b9a64d7000ce66c21"
              tar xvzf ds210_downsampled.tar.gz -C /tmp/data/
            else
              echo "Dataset ds000210 was cached"
            fi
      - run:
          name: Get FreeSurfer derivatives for ds000005
          command: |
            if [[ ! -d /tmp/ds005/freesurfer ]]; then
              mkdir -p /tmp/ds005
              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
                -O ds005_derivatives_freesurfer.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/58fe59eb594d900250960180"
              tar xvzf ds005_derivatives_freesurfer.tar.gz -C /tmp/ds005
            else
              echo "FreeSurfer derivatives of ds000005 were cached"
            fi
      - run:
          name: Store FreeSurfer license file
          command: |
            mkdir -p /tmp/fslicense
            cd /tmp/fslicense
            echo "<KEY> | base64 -d | sh
      - run:
          name: Create Nipype config files
          command: |
            mkdir -p /tmp/ds005 /tmp/ds054 /tmp/ds210
            printf "[execution]\nstop_on_first_crash = true\n" > /tmp/ds005/nipype.cfg
            echo "poll_sleep_duration = 0.01" >> /tmp/ds005/nipype.cfg
            echo "hash_method = content" >> /tmp/ds005/nipype.cfg
            cp /tmp/ds005/nipype.cfg /tmp/ds054/nipype.cfg
            cp /tmp/ds005/nipype.cfg /tmp/ds210/nipype.cfg
      - persist_to_workspace:
          root: /tmp
          paths:
            - fslicense
            - ds005/nipype.cfg
            - ds054/nipype.cfg
            - ds210/nipype.cfg
      - save_cache:
         key: data-v7-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
         paths:
            - /tmp/data
            - /tmp/ds005/freesurfer

  test_pytest:
    machine:
      image: circleci/classic:201711-01
    working_directory: /tmp/src/fmriprep
    steps:
      - checkout:
          path: /home/circleci/src/fmriprep
      - run:
          name: Check whether build should be skipped
          command: |
            cd /home/circleci/src/fmriprep
            if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
              echo "Only docs build"
              circleci step halt
            fi
            if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[skip[ _]?tests\]' )" != "" ]]; then
              echo "Skipping pytest job"
              circleci step halt
            fi

      - attach_workspace:
          at: /tmp
      - restore_cache:
          keys:
            - build-v4-{{ .Branch }}-{{ epoch }}
            - build-v4-{{ .Branch }}-
            - build-v4-
          paths:
            - /tmp/docker
      - restore_cache:
          keys:
            - data-v8-{{ .Revision }}
      - run:
          name: Set-up a Docker registry
          command: |
            docker run -d -p 5000:5000 --restart=always --name=registry \
                -v /tmp/docker:/var/lib/registry registry:2
      - run:
          name: Pull images from local registry
          command: |
            docker pull localhost:5000/fmriprep
            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
      - run:
          name: Run fMRIPrep tests
          no_output_timeout: 2h
          command: |
            docker run -ti --rm=false \
              -e TEST_READONLY_FILESYSTEM=1 -v $HOME:/home/readonly:ro \
              --entrypoint="pytest" poldracklab/fmriprep:latest \
              /src/fmriprep/fmriprep -svx --doctest-modules

      - run:
          name: Test fmriprep-wrapper (Python 2)
          command: |
            export PY2=$(pyenv versions | grep '2\.' |
                         sed -e 's/.* 2\./2./' -e 's/ .*//')
            pyenv local $PY2
            echo -n "Python version: "
            python --version
            pip install --upgrade pip setuptools
            pip install --upgrade wrapper/
            which fmriprep-docker
            fmriprep-docker -i poldracklab/fmriprep:latest --help
            fmriprep-docker -i poldracklab/fmriprep:latest --version
      - run:
          name: Test fmriprep-wrapper (Python 3)
          command: |
            export PY3=$(pyenv versions | grep '3\.' |
                         sed -e 's/.* 3\./3./' -e 's/ .*//')
            pyenv local $PY3
            echo -n "Python version: "
            python --version
            pip install --upgrade pip setuptools
            pip install --upgrade wrapper/
            which fmriprep-docker
            fmriprep-docker -i poldracklab/fmriprep:latest --help
            fmriprep-docker -i poldracklab/fmriprep:latest --version
      - store_artifacts:
          path: /tmp/data/reports

  build_docs:
    docker:
      - image: python:3.7.4
    working_directory: /tmp/src/fmriprep
    environment:
      - FSLOUTPUTTYPE: 'NIFTI'
    steps:
      - checkout
      - run:
          name: Check whether build should be skipped
          command: |
            if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[skip[ _]?docs\]' )" != "" ]]; then
              echo "Skipping doc building job"
              circleci step halt
            fi
      - run:
          name: Check Python version and upgrade pip
          command: |
            python --version
            python -m pip install -U pip
      - run:
          name: Install graphviz
          command: |
              apt-get update
              apt-get install -y graphviz
      - run:
          name: Install Requirements (may contain pinned versions)
          command: python -m pip install -r docs/requirements.txt
      - run:
          name: Install fMRIPrep
          command: python -m pip install ".[doc]"
      - run:
          name: Build documentation
          command: make SPHINXOPTS="-W" -C docs html
      - store_artifacts:
          path: /tmp/src/fmriprep/docs/_build/html

  ds005:
    machine:
      image: circleci/classic:201711-01
    working_directory: /tmp/ds005
    environment:
      - FS_LICENSE: /tmp/fslicense/license.txt
    steps:
      - checkout:
          path: /home/circleci/src/fmriprep
      - run:
          name: Check whether build should be skipped
          command: |
            cd /home/circleci/src/fmriprep
            if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
              echo "Only docs build"
              circleci step halt
            fi
            if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[skip[ _]?ds005\]' )" != "" ]]; then
              echo "Skipping ds000005 build"
              circleci step halt
            fi

      - attach_workspace:
          at: /tmp
      - restore_cache:
          keys:
            - build-v4-{{ .Branch }}-{{ epoch }}
            - build-v4-{{ .Branch }}-
            - build-v4-
          paths:
            - /tmp/docker
      - restore_cache:
          keys:
            - data-v7-{{ .Branch }}-{{ .Revision }}
      - restore_cache:
          keys:
            - ds005-anat-v17-{{ .Branch }}-{{ .Revision }}
            - ds005-anat-v17-{{ .Branch }}
            - ds005-anat-v17-master
            - ds005-anat-v17-
      - run:
          name: Set-up a Docker registry
          command: |
            docker run -d -p 5000:5000 --restart=always --name=registry \
                -v /tmp/docker:/var/lib/registry registry:2
      - run:
          name: Pull images from local registry
          command: |
            docker pull localhost:5000/fmriprep
            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
      - run:
          name: Setting up test
          command: |
            mkdir -p /tmp/ds005/derivatives && sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/derivatives && sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/derivatives
            pip install --upgrade pip setuptools
            pip install --upgrade /tmp/src/fmriprep/wrapper/
      - run:
          name: Run anatomical workflow on ds005
          no_output_timeout: 2h
          command: |
            mkdir -p /tmp/ds005/work /tmp/ds005/derivatives
            sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/derivatives && \
                sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/derivatives
            sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/work && \
                sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/work
            fmriprep-docker -i poldracklab/fmriprep:latest \
                -e FMRIPREP_DEV 1 -u $(id -u) \
                --network none \
                --config $PWD/nipype.cfg -w /tmp/ds005/work \
                /tmp/data/ds005 /tmp/ds005/derivatives participant \
                --fs-subjects-dir /tmp/ds005/freesurfer \
                --skull-strip-template OASIS30ANTs:res-1 \
                --output-spaces MNI152NLin2009cAsym MNI152NLin6Asym \
                --sloppy --write-graph --mem_mb 4096 \
                --nthreads 2 --anat-only -vv
      - run:
          name: Clean-up after anatomical run
          command: |
            rm -rf /tmp/ds005/work/fmriprep_wf/fsdir*
            rm -rf /tmp/ds005/work/reportlets
            rm -rf /tmp/ds005/derivatives/fmriprep
      - save_cache:
         key: ds005-anat-v17-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
         paths:
            - /tmp/ds005/work

      - run:
          name: Run full fMRIPrep on ds005 (LegacyMultiProc plugin)
          no_output_timeout: 2h
          command: |
            sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/derivatives && \
                sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/derivatives
            sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/work && \
                sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/work
            fmriprep-docker -i poldracklab/fmriprep:latest \
                -e FMRIPREP_DEV 1 -u $(id -u) \
                --network none \
                --config $PWD/nipype.cfg -w /tmp/ds005/work \
                /tmp/data/ds005 /tmp/ds005/derivatives participant \
                --fs-subjects-dir /tmp/ds005/freesurfer \
                --sloppy --write-graph --use-syn-sdc --mem_mb 4096 \
                --use-aroma \
                --skull-strip-template OASIS30ANTs:res-1 \
                --output-spaces MNI152NLin2009cAsym fsaverage5 fsnative MNI152NLin6Asym anat \
                --use-plugin /home/circleci/src/fmriprep/.circleci/legacy.yml \
                --nthreads 2 --cifti-output -vv
      - run:
          name: Move intermediate results for second run
          command: |
            mkdir -p /tmp/ds005/derivatives_partial
            sudo cp -a /tmp/ds005/work /tmp/ds005/work_partial
            sudo rm -rf /tmp/ds005/work_partial/fmriprep_wf/single_subject_01_wf/func_preproc_task_mixedgamblestask_run_02_wf/ica_aroma_wf
      - run:
          name: Checking outputs of full fMRIPrep run
          command: |
            mkdir -p /tmp/ds005/test
            find /tmp/ds005/derivatives -path */figures -prune -o -name "*" -print | sed s+/tmp/ds005/derivatives/++ | sort > /tmp/ds005/test/outputs.out
            diff /tmp/src/fmriprep/.circleci/ds005_outputs.txt /tmp/ds005/test/outputs.out
            exit $?
      - run:
          name: Re-run fMRIPrep on single run of task data
          no_output_timeout: 2h
          command: |
            rm /tmp/data/ds005/sub-01/func/*_run-01_*
            fmriprep-docker -i poldracklab/fmriprep:latest \
                -e FMRIPREP_DEV 1 -u $(id -u) \
                --network none \
                --config $PWD/nipype.cfg -w /tmp/ds005/work_partial \
                /tmp/data/ds005 /tmp/ds005/derivatives_partial participant \
                --fs-subjects-dir /tmp/ds005/freesurfer \
                --sloppy --write-graph --use-syn-sdc --mem_mb 4096 \
                --output-spaces MNI152NLin2009cAsym fsaverage5 fsnative MNI152NLin6Asym anat \
{'fraction_non_alphanumeric': 0.1171753677612446, 'fraction_numerical': 0.03961374481181353, 'mean_word_length': 1.4359009628610728, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 16, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3785184', 'n_tokens_mistral': 12532, 'n_tokens_neox': 10974, 'n_words': 2595}