From aa09328060b84c3522eea2d1fde14249467b96a7 Mon Sep 17 00:00:00 2001 From: Ino de Bruijn Date: Mon, 10 Oct 2016 03:12:00 -0400 Subject: [PATCH 1/5] conda-requirements.txt as env option, add setup.py install --- bin/steps/conda_compile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/steps/conda_compile b/bin/steps/conda_compile index 5ef775d..cab9ce6 100755 --- a/bin/steps/conda_compile +++ b/bin/steps/conda_compile @@ -12,9 +12,9 @@ echo "added pinned file in $HOME/.heroku/miniconda/conda-meta/pinned" conda install nomkl -if [ -f conda-requirements.txt ]; then +if [ -f ${CONDA_REQ_FILE:-conda-requirements.txt} ]; then puts-step "Installing dependencies using Conda" - conda install --file conda-requirements.txt --yes | indent + conda install --file ${CONDA_REQ_FILE:-conda-requirements.txt} --yes | indent fi if [ -f requirements.txt ]; then @@ -22,5 +22,9 @@ if [ -f requirements.txt ]; then pip install -r requirements.txt --exists-action=w --allow-all-external | indent fi +if [ -f setup.py ]; then + pip install --exists-action=w --allow-all-external -e . | indent +fi + # Clean up the installation environment . conda clean -pt --yes > /dev/null From 68eecb4942491a4a9de3afe5c634c3a6e8f712af Mon Sep 17 00:00:00 2001 From: Ino de Bruijn Date: Mon, 10 Oct 2016 03:20:56 -0400 Subject: [PATCH 2/5] Add channels env var --- bin/steps/conda_compile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/steps/conda_compile b/bin/steps/conda_compile index cab9ce6..53a8f85 100755 --- a/bin/steps/conda_compile +++ b/bin/steps/conda_compile @@ -11,6 +11,11 @@ echo "nomkl" > $HOME/.heroku/miniconda/conda-meta/pinned echo "added pinned file in $HOME/.heroku/miniconda/conda-meta/pinned" conda install nomkl +if [ -n "${CONDA_CHANNELS}" ]; then + for channel in ${CONDA_CHANNELS}; do + conda config --add channels $channel + done +fi if [ -f ${CONDA_REQ_FILE:-conda-requirements.txt} ]; then puts-step "Installing dependencies using Conda" From aff0b0d646252cca6c402cba78af3fdc48207204 Mon Sep 17 00:00:00 2001 From: Ino de Bruijn Date: Mon, 10 Oct 2016 11:24:06 -0400 Subject: [PATCH 3/5] Add logging --- bin/steps/conda_compile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/steps/conda_compile b/bin/steps/conda_compile index 53a8f85..ac96d63 100755 --- a/bin/steps/conda_compile +++ b/bin/steps/conda_compile @@ -17,17 +17,20 @@ if [ -n "${CONDA_CHANNELS}" ]; then done fi +puts-step "Looking for conda deps in ${CONDA_REQ_FILE:-conda-requirements.txt}" if [ -f ${CONDA_REQ_FILE:-conda-requirements.txt} ]; then - puts-step "Installing dependencies using Conda" + puts-step "Installing dependencies using Conda form ${CONDA_REQ_FILE}" conda install --file ${CONDA_REQ_FILE:-conda-requirements.txt} --yes | indent fi +puts-step "Looking for deps in requirements.txt" if [ -f requirements.txt ]; then - puts-step "Installing dependencies using Pip" + puts-step "Installing dependencies using Pip from requirements.txt" pip install -r requirements.txt --exists-action=w --allow-all-external | indent fi if [ -f setup.py ]; then + puts-step "Installing setup.py using pip install -e ." pip install --exists-action=w --allow-all-external -e . | indent fi From b70ed5f6e7588746e67ce8899ea4c170494df844 Mon Sep 17 00:00:00 2001 From: Ino de Bruijn Date: Mon, 10 Oct 2016 11:35:03 -0400 Subject: [PATCH 4/5] Add ENV_DIR parsing --- bin/compile | 9 +++++++-- bin/utils | 13 +++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index baf25e5..ac0ccb0 100755 --- a/bin/compile +++ b/bin/compile @@ -2,7 +2,7 @@ # Usage: # -# $ bin/compile +# $ bin/compile # Fail fast and fail hard. @@ -16,6 +16,7 @@ BIN_DIR=$(cd $(dirname $0); pwd) # absolute path ROOT_DIR=$(dirname $BIN_DIR) BUILD_DIR=$1 CACHE_DIR=$2 +ENV_DIR=$3 CACHED_DIRS=".heroku" @@ -31,6 +32,7 @@ export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH # Syntax sugar. source $BIN_DIR/utils + # Directory Hacks for path consistiency. APP_DIR='/app' TMP_APP_DIR=$CACHE_DIR/tmp_app_dir @@ -73,7 +75,10 @@ set -e # Make profile.d directory. mkdir -p $(dirname $PROFILE_PATH) -# Actuall do the conda steps. +# Add environment variables +export_env_dir $ENV_DIR + +# Actually do the conda steps. source $BIN_DIR/steps/conda_compile # ### Finalize diff --git a/bin/utils b/bin/utils index d849e0c..e9d06d7 100755 --- a/bin/utils +++ b/bin/utils @@ -52,3 +52,16 @@ function deep-mv (){ rm -fr $1/!(tmp) find -H $1 -maxdepth 1 -name '.*' -a \( -type d -o -type f -o -type l \) -exec rm -fr '{}' \; } + +export_env_dir() { + env_dir=$1 + whitelist_regex=${2:-''} + blacklist_regex=${3:-'^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH|JAVA_OPTS)$'} + if [ -d "$env_dir" ]; then + for e in $(ls $env_dir); do + echo "$e" | grep -E "$whitelist_regex" | grep -qvE "$blacklist_regex" && + export "$e=$(cat $env_dir/$e)" + : + done + fi +} From ad41836533a49fdb40f95ca65acfd068bc395058 Mon Sep 17 00:00:00 2001 From: Ino de Bruijn Date: Mon, 10 Oct 2016 11:59:53 -0400 Subject: [PATCH 5/5] Add conda channel logging --- bin/compile | 1 + bin/steps/conda_compile | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index ac0ccb0..476fe6f 100755 --- a/bin/compile +++ b/bin/compile @@ -78,6 +78,7 @@ mkdir -p $(dirname $PROFILE_PATH) # Add environment variables export_env_dir $ENV_DIR + # Actually do the conda steps. source $BIN_DIR/steps/conda_compile diff --git a/bin/steps/conda_compile b/bin/steps/conda_compile index ac96d63..bbc90b9 100755 --- a/bin/steps/conda_compile +++ b/bin/steps/conda_compile @@ -9,14 +9,16 @@ fi echo "nomkl" > $HOME/.heroku/miniconda/conda-meta/pinned echo "added pinned file in $HOME/.heroku/miniconda/conda-meta/pinned" -conda install nomkl if [ -n "${CONDA_CHANNELS}" ]; then for channel in ${CONDA_CHANNELS}; do + puts-step "Add conda channel $channel" conda config --add channels $channel done fi +conda install nomkl + puts-step "Looking for conda deps in ${CONDA_REQ_FILE:-conda-requirements.txt}" if [ -f ${CONDA_REQ_FILE:-conda-requirements.txt} ]; then puts-step "Installing dependencies using Conda form ${CONDA_REQ_FILE}"