From 5b88ca070bfbb5d69d650f025985c4c0ba5fee8c Mon Sep 17 00:00:00 2001
From: Amir MOHAMMADI <amir.mohammadi@idiap.ch>
Date: Tue, 26 Sep 2017 18:47:04 +0200
Subject: [PATCH] Merge conda caches
---
gitlab/build.sh | 28 ++++------------------------
gitlab/functions.sh | 34 +++++++++++++++++++---------------
2 files changed, 23 insertions(+), 39 deletions(-)
diff --git a/gitlab/build.sh b/gitlab/build.sh
index 18a6480..8ae5502 100755
--- a/gitlab/build.sh
+++ b/gitlab/build.sh
@@ -2,25 +2,8 @@
# Wed 21 Sep 2016 13:08:05 CEST
source $(dirname ${0})/functions.sh
-# look into possible cache folders
-log_info ls ${CONDA_FOLDER}/pkgs
-ls ${CONDA_FOLDER}/pkgs
-log_info ls ${CONDA_ENVS_PATH}
-ls ${CONDA_ENVS_PATH}
-log_info ls -R /
-ls -R /
-# look into possible cache folders
+
run_cmd ${CONDA_FOLDER}/bin/conda install -n root --yes --quiet conda=4 conda-build=3
-# look into possible cache folders
-log_info ls ${CONDA_FOLDER}/pkgs
-ls ${CONDA_FOLDER}/pkgs
-log_info ls ${CONDA_ENVS_PATH}
-ls ${CONDA_ENVS_PATH}
-log_info ls /home/dev/.conda/pkgs
-ls /home/dev/.conda/pkgs
-log_info ls -R /
-ls -R /
-# look into possible cache folders
run_cmd ${CONDA_FOLDER}/bin/conda config --set always_yes yes --set changeps1 no
run_cmd ${CONDA_FOLDER}/bin/conda config --set show_channel_urls true
run_cmd ${CONDA_FOLDER}/bin/conda clean --lock
@@ -44,9 +27,6 @@ if [ -z "${CI_COMMIT_TAG}" ]; then
else
run_cmd ${CONDA_FOLDER}/bin/conda build --override-channels -c ${CONDA_CHANNEL} -c defaults --python=${PYTHON_VERSION} conda
fi
-# look into possible cache folders
-log_info ls ${CONDA_FOLDER}/pkgs
-ls ${CONDA_FOLDER}/pkgs
-log_info ls ${CONDA_ENVS_PATH}
-ls ${CONDA_ENVS_PATH}
-# look into possible cache folders
+
+# Copy the .tar.bz2 and urls.txt
+merge_conda_cache ${CONDA_ENVS_PATH}/.pkgs ${CONDA_FOLDER}/pkgs
diff --git a/gitlab/functions.sh b/gitlab/functions.sh
index fa1df62..90c1623 100644
--- a/gitlab/functions.sh
+++ b/gitlab/functions.sh
@@ -264,6 +264,22 @@ if [ -z "${BOB_PACKAGE_VERSION}" ]; then
fi
+# merges conda cache folders
+# $1: Path to the main cache to keep. The directory must exist.
+# $2: Path to the extra cache to be merged into main cache
+merge_conda_cache() {
+ if [ -e ${1} ]; then
+ _cached_urls="${2}/urls.txt"
+ _urls="${1}/urls.txt"
+ if [ -e ${2} ]; then
+ log_info "Merging urls.txt and packages with cached files..."
+ mv -n ${2}/*.tar.bz2 ${1}/
+ cat ${_urls} ${_cached_urls} | sort | uniq > ${_urls}
+ fi
+ fi
+}
+
+
# installs a miniconda installation.
# $1: Path to where to install miniconda.
install_miniconda() {
@@ -290,25 +306,13 @@ install_miniconda() {
run_cmd mv ${1} ${1}.cached
fi
- # # Save urls.txt to create a valid cache after conda
- # # installation override
- # if [ -e ${_urls} ]; then
- # run_cmd cp ${_urls} ${_urls}.cached
- # fi
-
# install miniconda
bash miniconda.sh -b -p ${1}
# Put back cache and merge urls.txt
- _cached_urls="${1}.cached/pkgs/urls.txt"
- _urls="${1}/pkgs/urls.txt"
- if [ -e ${1}.cached ]; then
- log_info "Merging urls.txt files with cached values..."
- mv -n ${1}.cached/pkgs/*.tar.bz2 ${1}/pkgs/
- cat ${_urls} ${_cached_urls} | sort | uniq > ${_urls}
- # remove the backup cache folder
- rm -rf ${1}.cached
- fi
+ merge_conda_cache ${1}/pkgs ${1}.cached/pkgs
+ # remove the backup cache folder
+ rm -rf ${1}.cached
# List currently available packages on cache
# run_cmd ls -l ${1}/pkgs/
--
GitLab