Skip to content

Commit 162dd7c

Browse files
satrasatra
authored
Merge pull request #1541 from oesteban/tst/Docker
[TST] Migrate CircleCI tests to docker
2 parents bd36a5d + ffc06e9 commit 162dd7c

16 files changed

+402
-272
lines changed

‎.noserc

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
[nosetests]
22
verbosity=3
33

4+
logging-level=DEBUG
5+
with-doctest=1
6+
with-xunit=1
47
with-coverage=1
58
cover-branches=1
69
cover-xml=1
7-
cover-xml-file=./coverage.xml
810
cover-min-percentage=50
9-
10-
11-
with-xunit=1

‎docker/test-image/Dockerfile renamed to ‎Dockerfile

+42-19
Original file line numberDiff line numberDiff line change
@@ -26,26 +26,49 @@
2626
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
2727
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2828

29-
FROM nipype/testnipype
29+
FROM nipype/testnipypedata:latest
3030
MAINTAINER Stanford Center for Reproducible Neuroscience <[email protected]>
3131

3232
# Preparations
3333
RUN ln -snf /bin/bash /bin/sh
34-
WORKDIR /root
35-
36-
RUN mkdir -p ~/examples/ && ln -sf /usr/share/fsl-feeds/ ~/examples/feeds
37-
RUN curl -sSL "https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2" && \
38-
tar jxvf nipype-tutorial.tar.bz2 && \
39-
mv nipype-tutorial/* ~/examples/ && \
40-
curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" && \
41-
curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" && \
42-
curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" && \
43-
mkdir ~/examples/fsl_course_data && \
44-
tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data && \
45-
tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data && \
46-
tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data && \
47-
echo 'export FSL_COURSE_DATA=/root/examples/fsl_course_data' >> /etc/profile.d/crn_neuro.sh && \
48-
echo 'export FSL_COURSE_DATA=/root/examples/fsl_course_data' >> /etc/bash.bashrc
49-
ENV FSL_COURSE_DATA /root/examples/fsl_course_data
50-
51-
CMD ["/bin/bash"]
34+
35+
# Install this branch's code
36+
WORKDIR /root/src
37+
38+
# Install matplotlib, sphinx and coverage to build documentation
39+
# and run tests with coverage
40+
RUN source activate nipypetests-2.7 && \
41+
pip install matplotlib sphinx coverage && \
42+
source activate nipypetests-3.4 && \
43+
pip install matplotlib sphinx coverage && \
44+
source activate nipypetests-3.5 && \
45+
pip install matplotlib sphinx coverage
46+
47+
ADD . nipype/
48+
49+
# Install the checked out version of nipype, check that requirements are
50+
# installed and install it for each of the three environments.
51+
RUN cd nipype/ && \
52+
source activate nipypetests-2.7 && \
53+
pip install -r requirements.txt && \
54+
pip install -e .
55+
56+
RUN cd nipype/ && \
57+
source activate nipypetests-3.4 && \
58+
pip install -r requirements.txt && \
59+
pip install -e .
60+
61+
RUN cd nipype/ && \
62+
source activate nipypetests-3.5 && \
63+
pip install -r requirements.txt && \
64+
pip install -e .
65+
66+
WORKDIR /scratch
67+
68+
# Install entrypoints
69+
ADD docker/circleci/run_* /usr/bin/
70+
RUN chmod +x /usr/bin/run_*
71+
72+
# RUN echo 'source /etc/profile.d/nipype_tests.sh' >> /etc/bash.bashrc
73+
ENTRYPOINT ["/usr/bin/run_examples.sh"]
74+

‎circle.yml

+35-67
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,46 @@
1+
machine:
2+
services:
3+
- docker
4+
15
dependencies:
26
cache_directories:
3-
- "~/.apt-cache"
4-
- "~/examples/data"
5-
- "~/examples/fsdata"
6-
- "~/mcr"
7-
- "~/spm12"
8-
- "~/examples/fsl_course_data"
7+
- "~/docker"
8+
99
pre:
10-
# Let CircleCI cache the apt archive
11-
- sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial
12-
- bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh)
10+
- mkdir -p "~/scratch/nose"
11+
1312
override:
14-
# Install apt packages
15-
- sudo apt-get install -y fsl-core fsl-atlases fsl-mni152-templates fsl-feeds afni swig python-vtk xvfb fusefat
16-
- echo 'source /etc/fsl/fsl.sh' >> $HOME/.profile
17-
- echo 'source /etc/afni/afni.sh' >> $HOME/.profile
18-
- mkdir -p ~/examples/ && ln -sf /usr/share/fsl-feeds/ ~/examples/feeds
19-
# Enable system-wide vtk
20-
- ln -sf /usr/lib/pymodules/python2.7/vtk ~/virtualenvs/venv-system/lib/python2.7/site-packages/
21-
# Set up python environment
22-
- pip install --upgrade pip
23-
- pip install -e .
24-
- pip install matplotlib sphinx ipython boto coverage dipy
25-
# Add tvtk
26-
- pip install http://effbot.org/downloads/Imaging-1.1.7.tar.gz
27-
- pip install -e git+https://github.com/enthought/etsdevtools.git#egg=etsdevtools
28-
- pip install -e git+https://github.com/enthought/blockcanvas.git#egg=blockcanvas
29-
- pip install -e git+https://github.com/enthought/etsproxy.git#egg=etsproxy
30-
- pip install -e git+https://github.com/enthought/ets.git#egg=ets
31-
- pip install https://github.com/rtfd/readthedocs-sphinx-ext/archive/master.zip
32-
- gem install fakes3
33-
- if [[ ! -d ~/examples/data ]]; then wget "https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2" && tar jxvf nipype-tutorial.tar.bz2 && mv nipype-tutorial/* ~/examples/; fi
34-
- if [[ ! -d ~/examples/fsl_course_data ]]; then wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" && mkdir ~/examples/fsl_course_data && tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data && tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data && tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data; fi
35-
- bash ~/nipype/tools/install_spm_mcr.sh
36-
- mkdir -p ~/.nipype && echo '[logging]' > ~/.nipype/nipype.cfg && echo 'workflow_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'interface_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'filemanip_level = DEBUG' >> ~/.nipype/nipype.cfg
37-
machine:
38-
environment:
39-
FSLOUTPUTTYPE: NIFTI_GZ
13+
- if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi
14+
- docker build -t nipype/testbench:latest . :
15+
timeout: 1600
16+
- mkdir -p ~/docker; docker save nipype/testbench:latest > ~/docker/image.tar :
17+
timeout: 1600
18+
4019
test:
4120
override:
42-
- mkdir -p ${CIRCLE_TEST_REPORTS}/nose
43-
- source $HOME/.profile; nosetests --with-doctest --xunit-file="${CIRCLE_TEST_REPORTS}/nose/${CIRCLE_PROJECT_REPONAME}.xml" -c ./.noserc --logging-level=DEBUG --verbosity=3:
44-
environment:
45-
SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
46-
FORCE_SPMMCR: 1
47-
FSL_COURSE_DATA: "$HOME/examples/fsl_course_data"
48-
timeout: 2600
49-
- set -o pipefail && cd doc && make html 2>&1 | tee ~/log.txt
50-
- cat ~/log.txt && if grep -q "ERROR" ~/log.txt; then false; else true; fi
51-
- source $HOME/.profile; python ~/nipype/tools/run_examples.py test_spm Linear workflow3d workflow4d:
52-
pwd: ../examples
53-
environment:
54-
SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
55-
FORCE_SPMMCR: 1
21+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch --entrypoint="/usr/bin/run_builddocs.sh" nipype/testbench
22+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench test_spm Linear /root/examples/ workflow3d :
5623
timeout: 1600
57-
- source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_fsl_feeds Linear l1pipeline:
58-
pwd: ../examples
59-
- source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_spm_dartel Linear level1 l2pipeline:
60-
pwd: ../examples
61-
environment:
62-
SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
63-
FORCE_SPMMCR: 1
24+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench test_spm Linear /root/examples/ workflow4d :
6425
timeout: 1600
65-
- source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_fsl_reuse Linear level1_workflow:
66-
pwd: ../examples
67-
- source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_spm_nested Linear level1 l2pipeline:
68-
pwd: ../examples
69-
environment:
70-
SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
71-
FORCE_SPMMCR: 1
26+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_fsl_feeds Linear /root/examples/ l1pipeline
27+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_dartel Linear /root/examples/ level1 :
28+
timeout: 1600
29+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_dartel Linear /root/examples/ l2pipeline :
30+
timeout: 1600
31+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_fsl_reuse Linear /root/examples/ level1_workflow
32+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_nested Linear /root/examples/ level1
33+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch nipype/testbench fmri_spm_nested Linear /root/examples/ l2pipeline
34+
- docker run -i -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /scratch --entrypoint="/usr/bin/run_nosetests.sh" nipype/testbench :
35+
timeout: 2600
36+
post:
37+
- bash docker/circleci/teardown.sh
7238

7339
general:
7440
artifacts:
75-
- "doc/_build/html"
76-
- "~/log.txt"
77-
- "nosetests.xml"
78-
- "coverage.xml"
41+
- "~/docs"
42+
- "~/logs"
43+
- "~/coverage.xml"
44+
- "~/nosetests.xml"
45+
- "~/builddocs.log"
46+
- "~/scratch"

‎docker/circleci/run_builddocs.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
for i in /etc/profile.d/*.sh; do
3+
source $i
4+
done
5+
source activate nipypetests-2.7
6+
7+
mkdir -p /scratch/docs
8+
set -o pipefail && cd /root/src/nipype/doc && make html 2>&1 | tee /scratch/builddocs.log
9+
cp -r /root/src/nipype/doc/_build/html/* /scratch/docs/
10+
chmod 777 -R /scratch/docs
11+
chmod 777 /scratch/builddocs.log
12+
cat /scratch/builddocs.log && if grep -q "ERROR" /scratch/builddocs.log; then false; else true; fi

‎docker/circleci/run_examples.sh

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
for i in /etc/profile.d/*.sh; do
3+
source $i
4+
done
5+
6+
mkdir -p /root/.nipype
7+
echo '[logging]' > /root/.nipype/nipype.cfg
8+
echo 'workflow_level = DEBUG' >> /root/.nipype/nipype.cfg
9+
echo 'interface_level = DEBUG' >> /root/.nipype/nipype.cfg
10+
echo 'filemanip_level = DEBUG' >> /root/.nipype/nipype.cfg
11+
12+
source activate nipypetests-2.7
13+
python /root/src/nipype/tools/run_examples.py $@

‎docker/circleci/run_nosetests.sh

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
for i in /etc/profile.d/*.sh; do
3+
source $i
4+
done
5+
source activate nipypetests-2.7
6+
cd /root/src/nipype
7+
mkdir -p /scratch/nose
8+
nosetests -c /root/src/nipype/.noserc --xunit-file="/scratch/nosetests.xml" --cover-xml-file="/scratch/coverage.xml"
9+
chmod 777 /scratch/nosetests.xml
10+
chmod 777 /scratch/coverage.xml
11+
chmod 777 -R /scratch/nose

‎docker/circleci/teardown.sh

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
#
3+
# This script puts all artifacts in place after the smoke tests
4+
#
5+
#
6+
set -u
7+
set -e
8+
9+
mkdir -p ${CIRCLE_TEST_REPORTS}/nose
10+
sudo mv ~/scratch/builddocs.log ~/builddocs.log
11+
sudo cp ~/scratch/nosetests.xml ${CIRCLE_TEST_REPORTS}/nose/${CIRCLE_PROJECT_REPONAME}.xml
12+
sudo mv ~/scratch/coverage.xml ~/coverage.xml
13+
mkdir -p ~/docs
14+
sudo mv ~/scratch/docs/* ~/docs/
15+
mkdir -p ~/logs
16+
sudo mv ~/scratch/logs/* ~/logs/

‎docker/test-image-base/Dockerfile

+3-3
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ RUN apt-get update && \
5757
libqhull-dev \
5858
libxft-dev \
5959
libjpeg-dev \
60-
libyaml-dev
60+
libyaml-dev \
61+
graphviz
6162

6263

6364
# Install ANTs
@@ -96,11 +97,10 @@ RUN echo "destinationFolder=/root/mcr" > mcr_options.txt && \
9697
unzip matlab_installer/installer.zip -d matlab_installer/ && \
9798
matlab_installer/install -inputFile mcr_options.txt && \
9899
rm -rf matlab_installer mcr_options.txt && \
99-
echo 'export LD_LIBRARY_PATH=/root/mcr/v85/runtime/glnxa64:/root/mcr/v85/bin/glnxa64:/root/mcr/v85/sys/os/glnxa64:$LD_LIBRARY_PATH' >> /etc/profile.d/nipype_deps.sh && \
100+
# echo 'export LD_LIBRARY_PATH=/root/mcr/v85/runtime/glnxa64:/root/mcr/v85/bin/glnxa64:/root/mcr/v85/sys/os/glnxa64:$LD_LIBRARY_PATH' >> /etc/profile.d/nipype_deps.sh && \
100101
echo 'export SPMMCRCMD="/root/spm12/run_spm12.sh /root/mcr/v85/ script"' >> /etc/profile.d/nipype_deps.sh && \
101102
echo 'export FORCE_SPMMCR=1' >> /etc/profile.d/nipype_deps.sh
102103

103-
ENV LD_LIBRARY_PATH "/root/mcr/v85/runtime/glnxa64:/root/mcr/v85/bin/glnxa64:/root/mcr/v85/sys/os/glnxa64:$LD_LIBRARY_PATH"
104104
ENV SPMMCRCMD "/root/spm12/run_spm12.sh /root/mcr/v85/ script"
105105
ENV FORCE_SPMMCR 1
106106

‎docker/test-image-data/Dockerfile

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Copyright (c) 2016, The developers of the Stanford CRN
2+
# All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# * Redistributions of source code must retain the above copyright notice, this
8+
# list of conditions and the following disclaimer.
9+
#
10+
# * Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# * Neither the name of crn_base nor the names of its
15+
# contributors may be used to endorse or promote products derived from
16+
# this software without specific prior written permission.
17+
#
18+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21+
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22+
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23+
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24+
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25+
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26+
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
29+
FROM nipype/testnipype:latest
30+
MAINTAINER Stanford Center for Reproducible Neuroscience <[email protected]>
31+
32+
# Preparations
33+
RUN ln -snf /bin/bash /bin/sh
34+
WORKDIR /root
35+
36+
# Install fsl-feeds and the nipype-tutorial data
37+
ARG DEBIAN_FRONTEND=noninteractive
38+
RUN apt-get update && \
39+
apt-get install -y fsl-feeds && \
40+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
41+
mkdir -p ~/examples/ && \
42+
ln -sf /usr/share/fsl-feeds/ ~/examples/feeds && \
43+
curl -sSL "https://dl.dropbox.com/s/jzgq2nupxyz36bp/nipype-tutorial.tar.bz2" -o nipype-tutorial.tar.bz2 && \
44+
tar jxf nipype-tutorial.tar.bz2 -C /root/examples
45+
46+
# RUN curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" -o fdt1.tar.gz && \
47+
# curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" -o fdt2.tar.gz && \
48+
# curl -sSL "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" -o tbss.tar.gz && \
49+
# mkdir ~/examples/fsl_course_data && \
50+
# echo 'Untarring fsl course data...' && \
51+
# tar zxf fdt1.tar.gz -C ~/examples/fsl_course_data && \
52+
# tar zxf fdt2.tar.gz -C ~/examples/fsl_course_data && \
53+
# tar zxf tbss.tar.gz -C ~/examples/fsl_course_data && \
54+
# echo 'export FSL_COURSE_DATA=/root/examples/fsl_course_data' >> /etc/profile.d/nipype_data.sh
55+
# ENV FSL_COURSE_DATA /root/examples/fsl_course_data
56+
# RUN echo 'source /etc/profile.d/nipype_data.sh' >> /etc/bash.bashrc
57+
58+
CMD ["/bin/bash"]

‎docker/test-image-nipype/Dockerfile

+9-6
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
2727
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2828

29-
FROM nipype/testbase
29+
FROM nipype/testbase:latest
3030
MAINTAINER Stanford Center for Reproducible Neuroscience <[email protected]>
3131

3232
# Preparations
@@ -53,18 +53,21 @@ RUN conda config --add channels conda-forge
5353
RUN conda create -y -n nipypetests-2.7 lockfile nipype && \
5454
echo '#!/bin/bash' >> /etc/profile.d/nipype.sh && \
5555
echo '#!/bin/bash' >> /etc/bashrc && \
56-
echo 'source activate nipypetests-2.7' >> /etc/profile.d/nipype.sh && \
57-
echo 'source activate nipypetests-2.7' >> /etc/bashrc
56+
echo 'source activate nipypetests-2.7' >> /etc/profile.d/nipype.sh
5857

5958
# Create conda environment
6059
RUN conda create -y -n nipypetests-3.4 lockfile nipype python=3.4
6160

6261
# Create conda environment
6362
RUN conda create -y -n nipypetests-3.5 lockfile nipype python=3.5
6463

65-
# Install pip
66-
# RUN source activate nipypetests-2.7 && \
67-
# pip install --upgrade pip && \
64+
# Install dipy
65+
RUN source activate nipypetests-2.7 && \
66+
pip install dipy && \
67+
source activate nipypetests-3.4 && \
68+
pip install dipy && \
69+
source activate nipypetests-3.5 && \
70+
pip install dipy
6871

6972
RUN echo "source /etc/profile.d/nipype.sh" >> /etc/bash.bashrc
7073
CMD ["/bin/bash"]

0 commit comments

Comments
 (0)