From 2739797f40f745d5a41201960f7826d0129ee297 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.anjos@idiap.ch>
Date: Wed, 21 Sep 2016 13:13:40 +0200
Subject: [PATCH] Cleanup, moving scripts from bob/snippets
---
README.md | 95 +--------------
bootstrap-conda.sh => ci/bootstrap.sh | 6 +-
ci/docs.sh | 26 ++++
ci/wheels.sh | 16 +++
install/README.md | 111 ++++++++++++++++++
.../conda-forge-keeplist.sh | 0
extra-wheels.sh => install/extra-wheels.sh | 0
from-list.sh => install/from-list.sh | 0
from-scratch.sh => install/from-scratch.sh | 0
install-conda.sh => install/install-conda.sh | 0
{linux => install/linux}/devel-py27.txt | 0
{linux => install/linux}/devel-py34.txt | 0
{linux => install/linux}/devel-py35.txt | 0
{macosx => install/macosx}/devel-py27.txt | 0
{macosx => install/macosx}/devel-py34.txt | 0
{macosx => install/macosx}/devel-py35.txt | 0
install/update-lists.sh | 28 +++++
upload-wheels.sh => install/upload-wheels.sh | 0
18 files changed, 191 insertions(+), 91 deletions(-)
rename bootstrap-conda.sh => ci/bootstrap.sh (94%)
create mode 100755 ci/docs.sh
create mode 100755 ci/wheels.sh
create mode 100644 install/README.md
rename conda-forge-keeplist.sh => install/conda-forge-keeplist.sh (100%)
rename extra-wheels.sh => install/extra-wheels.sh (100%)
rename from-list.sh => install/from-list.sh (100%)
rename from-scratch.sh => install/from-scratch.sh (100%)
rename install-conda.sh => install/install-conda.sh (100%)
rename {linux => install/linux}/devel-py27.txt (100%)
rename {linux => install/linux}/devel-py34.txt (100%)
rename {linux => install/linux}/devel-py35.txt (100%)
rename {macosx => install/macosx}/devel-py27.txt (100%)
rename {macosx => install/macosx}/devel-py34.txt (100%)
rename {macosx => install/macosx}/devel-py35.txt (100%)
create mode 100644 install/update-lists.sh
rename upload-wheels.sh => install/upload-wheels.sh (100%)
diff --git a/README.md b/README.md
index 933d509..d2085d5 100644
--- a/README.md
+++ b/README.md
@@ -1,95 +1,10 @@
# Recipes and Scripts for Bob Maintenance
-This package contains administration scripts for our wikis and the build
+This package contains administrative scripts for our wikis and the build
system. Use it and update it if you must.
+In the directory `install`, you'll find scripts and instructions related to the
+installation and maintenance of new build machines.
-## Basic conda installation
-
-If you still don't have a basic conda installation, use `install-conda.sh` to
-create a new installation:
-
-```sh
-$ ./install-conda.sh /opt/conda
-```
-
-The single parameter defines the prefix of the installation. You'll use this
-prefix with the next commands. If everything is already in place, calling this
-script only updates your conda root environment.
-
-
-## From scratch
-
-If you don't have any environment setup, or you want to have the current list
-of packages updated, use the `from-scratch.sh` script. It builds an
-environment pulling the latest versions of our dependencies from both the
-`defaults` and `conda-forge` channels:
-
-```sh
-$ ./from-scratch.sh /opt/conda bob-devel-py27 2.7
-```
-
-The parameters are:
-
- 1. The base directory where the environment should be installed
- 2. The name of the environment to be created
- 3. The version of python to use for this environment
-
-Once the environment is generated, you may generate the package list like this:
-
-```sh
-$ /opt/conda/bin/conda list -n bob-devel-py27 -e > linux/devel-py27.txt
-```
-
-
-## From list
-
-To create a fully functional environment from a file list, use the following:
-
-```sh
-$ ./from-list.sh /opt/conda bob-devel-py27 linux/devel-py27.txt
-```
-
-The parameters are:
-
- 1. The base directory where the environment should be installed
- 2. The name of the environment to be created
- 3. The package list file to use for the newly created environment
-
-
-#### Extra OS packages
-
-Besides the dependencies encoded in `from-scratch.sh`, Bob development also
-requires the following packages, you must procure for your own operating
-system:
-
- * git (required to checkout packages via `mr.developer`)
- * LaTeX (required by matplotlib legends and sphinx documentation generation):
- * texlive-latex-extra
- * texlive-latex-recommended
- * texlive-fonts-recommended
- * dvipng
- * gcc/clang compiler suite + standard c++ library
- * On Linux, use gcc + libstdc++. These are normally distributed by
- installing the gcc and/or g++ packages of your distribution
- * On Mac OSX, use clang + libc++ (compatibility with `conda-forge`). Clang
- and libc++ are distributed with XCode
-
-
-#### Extra pip packages
-
-From time to time, you'll need to install extra pip packages on your
-environment. The most efficient way to manage those is to install by hand on
-the environment, using `pip`, then generate a wheel that you'll upload to our
-wheel repository for later serving. This wheels will work with the Conda
-installation you'll use.
-
-To generate these wheels and upload to our server, do:
-
-```sh
-$ ./extra-wheels.sh /opt/conda bob-devel-py27
-$ ./extra-wheels.sh /opt/conda bob-devel-py34
-$ ./extra-wheels.sh /opt/conda bob-devel-py35
-$ ./upload-wheels.sh *.whl
-password: *********
-```
+In the directory `ci`, you'll find scripts that are used for our continuous
+integration builds.
diff --git a/bootstrap-conda.sh b/ci/bootstrap.sh
similarity index 94%
rename from bootstrap-conda.sh
rename to ci/bootstrap.sh
index 8cb80a5..f609717 100755
--- a/bootstrap-conda.sh
+++ b/ci/bootstrap.sh
@@ -41,9 +41,12 @@ run_cmd() {
if [ ! -d ${PREFIX} ]; then
echo "[++] Downloading environment list into file \`env.txt'..."
pyver=$(echo ${PYTHON_VERSION} | tr -d '.')
- curl --silent https://gitlab.idiap.ch/bob/bob.admin/raw/master/${ARCH}/devel-py${pyver}.txt > env.txt
+ curl --silent https://gitlab.idiap.ch/bob/bob.admin/raw/master/install/${ARCH}/devel-py${pyver}.txt > env.txt
echo "[++] Bootstrapping conda installation at ${PREFIX}..."
run_cmd ${CONDA_FOLDER}/bin/conda create --prefix ${PREFIX} --file env.txt --yes
+
+ # Dirty fix to libjpeg.8 compilation issues:
+ # https://github.com/ContinuumIO/anaconda-issues/issues/1042
if [ "${ARCH}" == "macosx" ] && [ -e "${PREFIX}/lib/libjpeg.8.dylib" ]; then
run_cmd install_name_tool -id @rpath/libjpeg.8.dylib ${PREFIX}/lib/libjpeg.8.dylib
fi
@@ -93,4 +96,5 @@ if [ -e bootstrap-buildout.py ]; then
run_cmd ${use_python} bootstrap-buildout.py
else
echo "[!!] No bootstrap-buildout.py file found, skipping 'buildout bootstrap'..."
+ exit 1
fi
diff --git a/ci/docs.sh b/ci/docs.sh
new file mode 100755
index 0000000..27bbbe7
--- /dev/null
+++ b/ci/docs.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Wed 21 Sep 2016 13:08:05 CEST
+
+if [ -z "${DOCUSER}" ] || [ -z "${DOCPASS}" ] || [ -z "${DOCSERVER}" ]; then
+ echo "DOCSERVER, DOCUSER and/or DOCPASS undefined - cannot upload";
+ exit 1
+fi
+
+info=sphinx/.gitlab-ci.info
+
+echo "repo=${CI_PROJECT_PATH}" > ${info}
+echo "branch=${CI_BUILD_REF_NAME}" >> ${info}
+echo "tag=${CI_BUILD_TAG}" >> ${info}
+echo "build=${CI_BUILD_ID}" >> ${info}
+echo "commit=${CI_BUILD_REF}" >> ${info}
+echo "runner=${CI_RUNNER_DESCRIPTION}" >> ${info}
+
+file=${CI_PROJECT_NAME}-${CI_BUILD_REF}.tar.bz2
+tar cfj ${filename} sphinx
+
+echo "[>>] Uploading ${file}..."
+curl --silent --user "${DOCUSER}:${DOCPASS}" --upload-file ${file} ${DOCSERVER}/software/bob/docs-upload/
+if [ $? != 0 ]; then
+ echo "[error] curl command finished with an error condition"
+ exit 1
+fi
diff --git a/ci/wheels.sh b/ci/wheels.sh
new file mode 100755
index 0000000..8d67c0d
--- /dev/null
+++ b/ci/wheels.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Wed 21 Sep 2016 13:06:56 CEST
+
+if [ -z "${DOCUSER}" ] || [ -z "${DOCPASS}" ] || [ -z "${DOCSERVER}" ]; then
+ echo "DOCSERVER, DOCUSER and/or DOCPASS undefined - cannot upload";
+ exit 1
+fi
+
+for file in dist/*.whl; do
+ echo "[>>] Uploading ${file}..."
+ curl --silent --user "${DOCUSER}:${DOCPASS}" --upload-file ${file} ${DOCSERVER}/software/bob/wheels-upload/gitlab/
+ if [ $? != 0 ]; then
+ echo "[error] curl command finished with an error condition"
+ exit 1
+ fi
+done
diff --git a/install/README.md b/install/README.md
new file mode 100644
index 0000000..786d1ce
--- /dev/null
+++ b/install/README.md
@@ -0,0 +1,111 @@
+# Recipes and Scripts for Bob Maintenance
+
+This package contains administration scripts for our wikis and the build
+system. Use it and update it if you must.
+
+
+## Basic conda installation
+
+If you still don't have a basic conda installation, use `install-conda.sh` to
+create a new installation:
+
+```sh
+$ ./install-conda.sh /opt/conda
+```
+
+The single parameter defines the prefix of the installation. You'll use this
+prefix with the next commands. If everything is already in place, calling this
+script only updates your conda root environment.
+
+
+## From scratch
+
+If you don't have any environment setup, or you want to have the current list
+of packages updated, use the `from-scratch.sh` script. It builds an
+environment pulling the latest versions of our dependencies from both the
+`defaults` and `conda-forge` channels:
+
+```sh
+$ ./from-scratch.sh /opt/conda bob-devel-py27 2.7
+```
+
+The parameters are:
+
+ 1. The base directory where the environment should be installed
+ 2. The name of the environment to be created
+ 3. The version of python to use for this environment
+
+Once the environment is generated, you may generate the package list like this:
+
+```sh
+$ /opt/conda/bin/conda list -n bob-devel-py27 -e > linux/devel-py27.txt
+```
+
+
+## From list
+
+To create a fully functional environment from a file list, use the following:
+
+```sh
+$ ./from-list.sh /opt/conda bob-devel-py27 linux/devel-py27.txt
+```
+
+The parameters are:
+
+ 1. The base directory where the environment should be installed
+ 2. The name of the environment to be created
+ 3. The package list file to use for the newly created environment
+
+
+#### Extra OS packages
+
+Besides the dependencies encoded in `from-scratch.sh`, Bob development also
+requires the following packages, you must procure for your own operating
+system:
+
+ * git (required to checkout packages via `mr.developer`)
+ * LaTeX (required by matplotlib legends and sphinx documentation generation):
+ * texlive-latex-extra
+ * texlive-latex-recommended
+ * texlive-fonts-recommended
+ * dvipng
+ * gcc/clang compiler suite + standard c++ library
+ * On Linux, use gcc + libstdc++. These are normally distributed by
+ installing the gcc and/or g++ packages of your distribution
+ * On Mac OSX, use clang + libc++ (compatibility with `conda-forge`). Clang
+ and libc++ are distributed with XCode
+
+
+#### Extra pip packages
+
+From time to time, you'll need to install extra pip packages on your
+environment. The most efficient way to manage those is to install by hand on
+the environment, using `pip`, then generate a wheel that you'll upload to our
+wheel repository for later serving. This wheels will work with the Conda
+installation you'll use.
+
+To generate these wheels and upload to our server, do:
+
+```sh
+$ ./extra-wheels.sh /opt/conda bob-devel-py27
+$ ./extra-wheels.sh /opt/conda bob-devel-py34
+$ ./extra-wheels.sh /opt/conda bob-devel-py35
+$ ./upload-wheels.sh *.whl
+password: *********
+```
+
+#### Creating or updating CI runners
+
+Run the `install-conda.sh` once to install a basic Conda support. To install or
+update our supported Python build environments, run `from-scratch.sh`. It will
+make the environment up-to-date with the current requirements for Bob.
+
+If an update was made, make sure to update the environment lists in `linux` and
+`macosx` sub-directories, as those are used as the basis for our builds. To do
+so, run:
+
+```sh
+$ ./update-lists.sh
+```
+
+and follow instructions.
diff --git a/conda-forge-keeplist.sh b/install/conda-forge-keeplist.sh
similarity index 100%
rename from conda-forge-keeplist.sh
rename to install/conda-forge-keeplist.sh
diff --git a/extra-wheels.sh b/install/extra-wheels.sh
similarity index 100%
rename from extra-wheels.sh
rename to install/extra-wheels.sh
diff --git a/from-list.sh b/install/from-list.sh
similarity index 100%
rename from from-list.sh
rename to install/from-list.sh
diff --git a/from-scratch.sh b/install/from-scratch.sh
similarity index 100%
rename from from-scratch.sh
rename to install/from-scratch.sh
diff --git a/install-conda.sh b/install/install-conda.sh
similarity index 100%
rename from install-conda.sh
rename to install/install-conda.sh
diff --git a/linux/devel-py27.txt b/install/linux/devel-py27.txt
similarity index 100%
rename from linux/devel-py27.txt
rename to install/linux/devel-py27.txt
diff --git a/linux/devel-py34.txt b/install/linux/devel-py34.txt
similarity index 100%
rename from linux/devel-py34.txt
rename to install/linux/devel-py34.txt
diff --git a/linux/devel-py35.txt b/install/linux/devel-py35.txt
similarity index 100%
rename from linux/devel-py35.txt
rename to install/linux/devel-py35.txt
diff --git a/macosx/devel-py27.txt b/install/macosx/devel-py27.txt
similarity index 100%
rename from macosx/devel-py27.txt
rename to install/macosx/devel-py27.txt
diff --git a/macosx/devel-py34.txt b/install/macosx/devel-py34.txt
similarity index 100%
rename from macosx/devel-py34.txt
rename to install/macosx/devel-py34.txt
diff --git a/macosx/devel-py35.txt b/install/macosx/devel-py35.txt
similarity index 100%
rename from macosx/devel-py35.txt
rename to install/macosx/devel-py35.txt
diff --git a/install/update-lists.sh b/install/update-lists.sh
new file mode 100644
index 0000000..15a3ffa
--- /dev/null
+++ b/install/update-lists.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Wed 21 Sep 2016 12:01:03 CEST
+
+if [ "${#}" != 1 ]; then
+ echo "usage: `basename $0` <prefix>"
+ echo "example: `basename $0` /opt/conda"
+ exit 1
+fi
+
+BASEDIR=$1
+CONDA=${BASEDIR}/bin/conda
+LISTDIR=`dirname $0`
+
+if [ "$(uname)" == "Darwin" ]; then
+ ARCH="macosx"
+else
+ ARCH="linux"
+fi
+
+for ver in py27 py34 py35; do
+ ${CONDA_ROOT}/bin/conda list -n "bob-devel-${ver}" --explict > "${LISTDIR}/${ARCH}/devel-${ver}.txt";
+done
+
+echo "I've changed the environment lists for ${ARCH} at ${LISTDIR}/${ARCH}"
+echo "You may now commit and push the results with something like:"
+echo "git commit -m \"Updated ${ARCH} environments\" -a"
+
+exit 0
diff --git a/upload-wheels.sh b/install/upload-wheels.sh
similarity index 100%
rename from upload-wheels.sh
rename to install/upload-wheels.sh
--
GitLab