Skip to content
Snippets Groups Projects
Commit 9b4d916d authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

Simple CI

parent 67e58a06
No related branches found
No related tags found
1 merge request!2Ci2
Pipeline #
Hide whitespace changes
Inline Side-by-side
Showing
with 61 additions and 863 deletions
.gitignore
+ 3
− 20
View file @ 9b4d916d
*~
*.swp
*.pyc
*.so
*.dylib
CMakeLists.txt
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
develop-eggs
sphinx
dist
.nfs*
.gdb_history
build
*.egg
opsnr.stt
.coverage
.DS_Store
_ci/
src/
dist/
.gitlab-ci.yml
+ 15
− 84
View file @ 9b4d916d
# This build file heavily uses template features from YAML so it is generic
# enough for any Bob project. Don't modify it unless you know what you're
# doing.
# Definition of our build pipeline
stages:
- build_bob
- test_bob
- wheels_bob
# ---------
# Templates
# ---------
- build
# Template for the build stage
# Needs to run on all supported architectures, platforms and python versions
.build_bob_template: &build_bob_job
stage: build_bob
core_linux_27:
stage: build
before_script:
- git clean -ffdx
#- mkdir _ci
#- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
#- chmod 755 _ci/install.sh
#- ./_ci/install.sh _ci #updates
- ./_ci/before_build.sh buildout-bob.cfg
script:
- ./_ci/build.sh buildout-bob.cfg requirements-bob.txt
after_script:
- ./_ci/after_build.sh
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/
# Template for the test stage - re-installs from uploaded wheels
# Needs to run on all supported architectures, platforms and python versions
.test_bob_template: &test_bob_job
stage: test_bob
before_script:
#- ./_ci/install.sh _ci #updates
- ./_ci/before_test.sh buildout-bob.cfg
script:
- ./_ci/test.sh buildout-bob.cfg
after_script:
- ./_ci/after_test.sh
# Template for the wheel uploading stage
# Needs to run against all combinations of python and operating systems
.wheels_bob_template: &wheels_bob_job
stage: wheels_bob
environment: intranet
only:
- new_ci
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
before_script:
#- ./_ci/install.sh _ci #updates
- ./_ci/before_wheels.sh
script:
- ./build.sh core.txt 2.7
- ./_ci/wheels.sh
after_script:
- ./_ci/after_wheels.sh
# -------------
# Build Targets
# -------------
# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
build_bob_linux_27:
<<: *build_bob_job
variables: &linux_27_build_variables
PYTHON_VERSION: "2.7"
tags:
- conda-linux
test_bob_linux_27:
<<: *test_bob_job
variables: *linux_27_build_variables
dependencies:
- build_bob_linux_27
- git clean -ffdx src
tags:
- conda-linux
wheels_bob_linux_27:
<<: *wheels_bob_job
variables: *linux_27_build_variables
extras_linux_27:
stage: build
dependencies:
- build_bob_linux_27
- core_linux_27
before_script:
- git clean -ffdx
script:
- ./build.sh extras.txt 2.7
- ./_ci/wheels.sh
after_script:
- git clean -ffdx
tags:
- conda-linux
LICENSE deleted 100644 → 0
+ 0
− 24
View file @ 67e58a06
Copyright (c) 2013, Andre Anjos - Idiap Research Institute
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer. Redistributions in binary
form must reproduce the above copyright notice, this list of conditions and the
following disclaimer in the documentation and/or other materials provided with
the distribution. Neither the name of the Idiap Research Institute nor the
names of its contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
README.rst
+ 43
− 96
View file @ 9b4d916d
.. vim: set fileencoding=utf-8 :
.. Andre Anjos <andre.anjos@idiap.ch>
.. Mon 03 Nov 2014 10:37:52 CET
.. Thu 22 Dec 2016 08:47:14 CET
.. |badge| image:: https://gitlab.idiap.ch/ci/projects/6/status.png?ref=master
:target: https://gitlab.idiap.ch/ci/projects/6?ref=master
.. image:: https://gitlab.idiap.ch/bob/bob.io.audio/badges/master/build.svg
:target: https://gitlab.idiap.ch/bob/bob.io.audio/commits/master
======================================
Bob Nightlies (concentrator) |badge|
======================================
===============
Bob Nightlies
===============
Bob is a free signal-processing and machine learning toolbox originally
developed by the Biometrics group at `Idiap`_ Research Institute, Switzerland.
This package is part of the signal-processing and machine learning toolbox
Bob_. This package implements the so-called "nightlies" aggregator. Its build
is triggered every night so all packages are properly recompiled against the
latest environment and tested together under that light.
The toolbox is written in a mix of `Python`_ and `C++`_ and is designed to be
both efficient and reduce development time. It is composed of a reasonably
large number of `packages`_ that implement tools for image, audio & video
processing, machine learning and pattern recognition.
**This package is not Bob. It is just a container that includes Bob
packages so we can continously integrate them.**
If just want to use Bob's functionalities on your experiments, you are **not**
supposed to install this package on your machine, but rather `create your own
personalised work environment
<https://github.com/idiap/bob/wiki/Installation>`_ depending on your needs, by
collecting individual sub-packages based on your requirements.
If you are developing Bob packages which are supposed to built along side our
`nightly build system <https://www.idiap.ch/software/bob/buildbot/waterfall>`_,
please read on.
Installation
------------
As per-usual, make sure all external `dependencies`_ are installed on your host
before trying to compile the whole of Bob. Once all dependencies_ are
satisfied, you should be able to::
$ python bootstrap.py
$ ./bin/buildout
You may tweak the options in ``buildout.cfg`` to disable/enable verbosity and
debug builds, **before you run** ``./bin/buildout``.
Documentation
-------------
You can generate the documentation for all packages in this container, after
installation, using Sphinx::
$ ./bin/sphinx-build . sphinx
This shall place in the directory ``sphinx``, the current version for the
documentation of the package.
Testing
-------
This package can be built starting from a base development environment, as
provided by our `from-scratch.sh`_ script. First run it to create a conda-based
environment for development. Once the environment is ready, follow the
instructions on the ``.gitlab-ci.yml`` file to complete the development cycle.
You can run a set of tests using the nose test runner::
The script ``build.sh`` will try to execute the steps from the CI in the right
order and can be used to locally test your modifications **before** pushing a
branch into the nightlies, which will trigger a complete set of builds in
multiple platforms.
$ ./bin/nosetests -sv
The script ``build.sh`` takes several parameters::
You can run our documentation tests using sphinx itself::
$ ./build.sh
$ ./bin/sphinx-build -b doctest . sphinx
Updating a Package
------------------
.. warning::
Before adding a package to this prototype, please ensure that the package:
Before adding a package to this prototype, please ensure that the package
contains all standard components of a Bob_ package (README, unit tests, CI
integration and documentation among others).
* contains a README clearly indicating how to install the package (including
external dependencies required). Also, please add package badges for the
build status and coverage as shown in other packages (even if your package
is not yet integrated to Travis or Coveralls).
If you don't know how to do this, ask for information on our `mailing
list`_.
* Has unit tests.
The new package sits in one of two categories:
* Is integrated with gitlab-ci and correctly tested on that platform (i.e.
it builds, it tests fine and a documentation can be constructed and tested
w/o errors)
1. If the package belongs to the core of Bob, add the package to the file
``core.txt``, following other packages already inserted. Be attentive to
the order in which packages should be compiled.
2. If the package is not part of the core of Bob, add the package to the file
``extras.txt``, following other packages already inserted. Be attentive
to the order in which packages should be compiled.
If you don't know how to do this, ask for information on the bob-devel
mailing list.
Once the file is modified, commit the changes to a **new branch**, with a
suggestive name. Push the branch and, once it is green, you may merge it to the
master (or ask it to be merged).
Then, add/update/remove the following files:
1. Add your package to the build::
$ vim layer2.cfg
2. Add your package to the overall documentation::
$ vim index.rst
3. Add your package to be "deploy-tested"::
$ vim requirements2.txt
4. Add a row for your package in our Packages_ list
5. Update the dependency graph::
$ tools/update_dependency_graphs.sh all
The current dependency graph looks like that:
.. image:: https://raw.githubusercontent.com/idiap/bob/master/dependencies.png
:target: https://raw.githubusercontent.com/idiap/bob/master/dependencies.png
:width: 50%
Contact
-------
For questions or reporting issues to this software package, contact our
development `mailing list`_.
.. External References
.. _c++: http://www2.research.att.com/~bs/C++.html
.. _python: http://www.python.org
.. _idiap: http://www.idiap.ch
.. _packages: https://gitlab.idiap.ch/bob/bob/wikis/Packages
.. _wiki: https://github.com/idiap/bob/wiki
.. _bug tracker: https://github.com/idiap/bob/issues
.. _dependencies: https://github.com/idiap/bob/wiki/Dependencies
.. Place your references here:
.. _bob: https://www.idiap.ch/software/bob
.. _mailing list: https://groups.google.com/forum/?fromgroups#!forum/bob-devel
.. _from-scratch.sh: https://gitlab.idiap.ch/bob/bob.admin/blob/master/install/from-scratch.sh
_ci/README.md deleted 100644 → 0
+ 0
− 23
View file @ 67e58a06
# Scripts for Continous Integration with Gitlab
This directory contains scripts that are used by our continuous integration
(CI) builds. There are various sets of scripts named as follows:
* `before_<stage>.sh`
* `<stage>.sh`
* `after_<stage>.sh`
Each of these sets correpond to one of the build stages inside the CI builds.
The script `install.sh` contains a basic set of routines to install and update
the scripts on the target host. If you add more scripts, it is sensible to
add those in there.
The script `functions.sh` contains a set of functions that are shared between
all scripts. All scripts *source* `functions.sh` before anything is done.
`functions.sh` also set important local and global (export) variables that are
required by the build system.
Before changing any script, try to understand the role of each of the stages by
looking at our template CI yaml files (in the directory `../templates`) and
then how to use the functions declared in `functions.sh`.
_ci/after_build.sh deleted 100755 → 0
+ 0
− 6
View file @ 67e58a06
#!/usr/bin/env bash
# Sat 24 Sep 08:12:24 2016 CEST
source $(dirname ${0})/functions.sh
log_info "*** Not yet implemented ***"
_ci/after_deploy.sh deleted 100755 → 0
+ 0
− 6
View file @ 67e58a06
#!/usr/bin/env bash
# Thu 22 Sep 2016 15:11:53 CEST
source $(dirname ${0})/functions.sh
log_info "*** Not yet implemented ***"
_ci/after_docs.sh deleted 100755 → 0
+ 0
− 6
View file @ 67e58a06
#!/usr/bin/env bash
# Thu 22 Sep 2016 18:23:57 CEST
source $(dirname ${0})/functions.sh
log_info "*** Not yet implemented ***"
_ci/after_test.sh deleted 100755 → 0
+ 0
− 6
View file @ 67e58a06
#!/usr/bin/env bash
# Thu 22 Sep 2016 13:59:03 CEST
source $(dirname ${0})/functions.sh
log_info "*** Not yet implemented ***"
_ci/after_wheels.sh deleted 100755 → 0
+ 0
− 6
View file @ 67e58a06
#!/usr/bin/env bash
# Thu 22 Sep 2016 18:23:57 CEST
source $(dirname ${0})/functions.sh
log_info "*** Not yet implemented ***"
_ci/before_build.sh deleted 100755 → 0
+ 0
− 73
View file @ 67e58a06
#!/usr/bin/env bash
# Mon 8 Aug 17:40:24 2016 CEST
source $(dirname ${0})/functions.sh
WHEELS_REPOSITORY="${DOCSERVER}/software/bob/wheels/gitlab/"
WHEELS_SERVER=`echo ${DOCSERVER} | sed 's;https\?://;;'`
check_env WHEELS_REPOSITORY
check_env WHEELS_SERVER
NIGHTLIES=$1
# Readies the virtualenv to use for installation
if [ ! -d ${PREFIX} ]; then
log_info "Creating virtualenv installation at ${PREFIX}..."
run_cmd ${BOB_PREFIX_PATH}/bin/virtualenv --system-site-packages ${PREFIX}
else
log_warn "Prefix directory ${PREFIX} exists, not re-installing..."
fi
# Source the newly created virtualenv
log_info "$ source ${PREFIX}/bin/activate"
source ${PREFIX}/bin/activate
# Verify where pip is installed
use_pip=`which pip`
if [ -z "${use_pip}" ]; then
log_error "Cannot find pip, aborting..."
exit 1
else
log_info "Using pip: ${use_pip}"
fi
# Upgrade wheel generation library to avoid bugs
run_cmd ${use_pip} install --upgrade setuptools pip wheel
use_python=`which python`
if [ -z "${use_python}" ]; then
log_error "Cannot find python, aborting..."
exit 1
else
log_info "Using python: ${use_python}"
fi
# Install this package's build dependencies
PIPOPTS="--find-links ${WHEELS_REPOSITORY} --trusted-host ${WHEELS_SERVER}"
PIPOPTS="${PIPOPTS} --pre --use-wheel --no-index"
if [ -e requirements.txt ]; then
run_cmd ${use_pip} install ${PIPOPTS} --requirement requirements.txt
else
log_info "No requirements.txt file found, skipping 'pip install <build-deps>'..."
fi
# Install this package's test dependencies
if [ -e test-requirements.txt ]; then
run_cmd ${use_pip} install ${PIPOPTS} --requirement test-requirements.txt
else
log_info "No test-requirements.txt file found, skipping 'pip install <test-deps>'..."
fi
# Finally, bootstrap the installation from the new environment
if [ -e bootstrap-buildout.py ]; then
if [ "$NIGHTLIES" -eq '' ]; then
run_cmd ${use_python} bootstrap-buildout.py
else
run_cmd ${use_python} bootstrap-buildout.py -c ${NIGHTLIES}
fi
else
log_error "No bootstrap-buildout.py file found, stopping..."
exit 1
fi
_ci/before_deploy.sh deleted 100755 → 0
+ 0
− 15
View file @ 67e58a06
#!/usr/bin/env bash
# Wed 21 Sep 2016 13:08:05 CEST
source $(dirname ${0})/functions.sh
run_cmd $(dirname ${0})/before_test.sh
# setup database locally and run `bob_dbmanage.py all download`
# if this is a database package - need auxiliary file for package
if [[ ${CI_PROJECT_NAME} == bob.db.* ]]; then
run_cmd ./bin/buildout
if [ -x ./bin/bob_dbmanage.py ]; then
run_cmd ./bin/bob_dbmanage.py all download --force;
fi
fi
_ci/before_docs.sh deleted 100755 → 0
+ 0
− 6
View file @ 67e58a06
#!/usr/bin/env bash
# Thu 22 Sep 2016 18:23:57 CEST
source $(dirname ${0})/functions.sh
log_info "*** Not yet implemented ***"
_ci/before_test.sh deleted 100755 → 0
+ 0
− 32
View file @ 67e58a06
#!/usr/bin/env bash
# Wed 21 Sep 2016 13:08:05 CEST
source $(dirname ${0})/functions.sh
NIGHTLIES=$1
if [ "$NIGHTLIES" -eq '' ]; then
run_cmd $(dirname ${0})/before_build.sh
else
run_cmd $(dirname ${0})/before_build.sh ${NIGHTLIES}
fi
# Source the newly created virtualenv
log_info "$ source ${PREFIX}/bin/activate"
source ${PREFIX}/bin/activate
# Verify where pip is installed
use_pip=`which pip`
if [ -z "${use_pip}" ]; then
log_error "Cannot find pip, aborting..."
exit 1
else
log_info "Using pip: ${use_pip}"
fi
# zc.recipe.egg needs some special installation instructions
if [ "${CI_PROJECT_NAME}" == "bob.buildout" ]; then
run_cmd ${use_pip} install --upgrade --no-binary ":all:" zc.recipe.egg
fi
run_cmd ${use_pip} install --upgrade --use-wheel --no-index --pre dist/*.whl
_ci/before_wheels.sh deleted 100755 → 0
+ 0
− 6
View file @ 67e58a06
#!/usr/bin/env bash
# Thu 22 Sep 2016 18:23:57 CEST
source $(dirname ${0})/functions.sh
log_info "*** Not yet implemented ***"
_ci/build.sh deleted 100755 → 0
+ 0
− 62
View file @ 67e58a06
#!/usr/bin/env bash
# Wed 21 Sep 2016 13:08:05 CEST
source $(dirname ${0})/functions.sh
NIGHTLIES=$1
REQUIREMENTS=$2
if [ "$NIGHTLIES" -eq '' ]; then
run_cmd ./bin/buildout
else
run_cmd ./bin/buildout -c ${NIGHTLIES}
fi
if [ -x ./bin/bob_dbmanage.py ]; then
run_cmd ./bin/bob_dbmanage.py all download --force;
fi
if [ -d ./doc ]; then
run_cmd ./bin/sphinx-build -W doc sphinx
fi
if [ "$NIGHTLIES" -eq '' ]; then
if [ -z "${WHEEL_TAG}" ]; then
# C/C++ extensions
run_cmd ./bin/python setup.py bdist_wheel
else
# Python-only packages
run_cmd ./bin/python setup.py bdist_wheel --python-tag ${WHEEL_TAG}
fi
else
# Generating the whell of the nightlies
if [ -z "${WHEEL_TAG}" ]; then
run_cmd ./bin/python setup.py bdist_wheel --dist-dir ./dist
else
run_cmd ./bin/python setup.py bdist_wheel --dist-dir ./dist --python-tag ${WHEEL_TAG}
fi
#WHEELS FROM THE DEPENDENCIES
for f in ${REQUIREMENTS}
do
while read -r line
do
dir="src/"$line
run_cmd cd $dir
if [ -z "${WHEEL_TAG}" ]; then
run_cmd ../../bin/python setup.py bdist_wheel --dist-dir ../../dist
else
run_cmd ../../bin/python setup.py bdist_wheel --dist-dir ../../dist --python-tag ${WHEEL_TAG}
fi
run_cmd cd ../../
done < "$f"
done
fi
_ci/deploy.sh deleted 100755 → 0
+ 0
− 59
View file @ 67e58a06
#!/usr/bin/env bash
# Thu 22 Sep 2016 13:59:03 CEST
source $(dirname ${0})/functions.sh
lock_pypirc
setup_deploy register --repository staging
setup_deploy check sdist --formats zip upload --repository staging
# if that worked, uploads source package to the production index
log_info "Uploading package to ${PYPISERVER} on behalf of ${PYPIUSER}..."
setup_deploy register --repository production
setup_deploy check sdist --formats zip upload --repository production
# if that worked, uploads documentation to pythonhosted if any exists
if [ -d sphinx ]; then
log_info "Uploading documentation to ${PYPISERVER} on behalf of ${PYPIUSER}..."
setup_deploy upload_docs --upload-dir sphinx --repository production
fi
unlock_pypirc
condaforge_packages=("bob" \
"bob.extension" \
"bob.blitz" \
"bob.core" \
"bob.ip.draw" \
"bob.io.base" \
"bob.sp" \
"bob.math" \
"bob.ap" \
"bob.measure" \
"bob.db.base" \
"bob.io.image" \
"bob.io.video" \
"bob.io.matlab" \
"bob.ip.base" \
"bob.ip.color" \
"bob.ip.gabor" \
"bob.learn.activation" \
"bob.learn.libsvm" \
"bob.learn.boosting" \
"bob.io.audio" \
"bob.learn.linear" \
"bob.learn.mlp" \
"bob.db.wine" \
"bob.db.mnist" \
"bob.db.atnt" \
"bob.ip.flandmark" \
"bob.ip.facedetect" \
"bob.ip.optflow.hornschunck" \
"bob.ip.optflow.liu" \
"bob.learn.em" \
"bob.db.iris")
if contains_element ${CI_PROJECT_NAME} "${condaforge_packages[@]}"; then
run_cmd ${CONDA_FOLDER}/bin/python _ci/update_feedstock.py ${CI_PROJECT_NAME}
fi
_ci/docs.sh deleted 100755 → 0
+ 0
− 18
View file @ 67e58a06
#!/usr/bin/env bash
# Wed 21 Sep 2016 13:08:05 CEST
source $(dirname ${0})/functions.sh
info=sphinx/.gitlab-ci.info
echo "repo=${CI_PROJECT_PATH}" > ${info}
echo "branch=${CI_BUILD_REF_NAME}" >> ${info}
echo "tag=${CI_BUILD_TAG}" >> ${info}
echo "build=${CI_BUILD_ID}" >> ${info}
echo "commit=${CI_BUILD_REF}" >> ${info}
echo "runner=${CI_RUNNER_DESCRIPTION}" >> ${info}
file=${CI_PROJECT_NAME}-${CI_BUILD_REF}.tar.bz2
run_cmd tar cfj ${file} sphinx
doc_upload ${file} software/bob/docs-upload/
_ci/functions.sh deleted 100644 → 0
+ 0
− 255
View file @ 67e58a06
#!/usr/bin/env bash
# Thu 22 Sep 2016 13:05:54 CEST
# Build utilities
# Determines the architecture we're using
arch() {
[[ "$(uname -s)" == "Darwin" ]] && echo "macosx" || echo "linux"
}
# Functions for coloring echo commands
log_info() {
echo -e "(`date +%T`) \033[1;34m${@}\033[0m"
}
log_warn() {
echo -e "(`date +%T`) \033[1;35mWarning: ${@}\033[0m" >&2
}
log_error() {
echo -e "(`date +%T`) \033[1;31mError: ${@}\033[0m" >&2
}
# Checks a given environment variable is set (non-zero size)
check_env() {
if [ -z "${!1}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
else
log_info "${1}=${!1}";
fi
}
# Exports a given environment variable, verbosely
export_env() {
if [ -z "${!1}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
else
export ${1}
log_info "export ${1}=${!1}";
fi
}
# Checks a given environment variable is set (non-zero size)
check_pass() {
if [ -z "${!1}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
else
log_info "${1}=********";
fi
}
# Function for running command and echoing results
run_cmd() {
log_info "$ ${@}"
${@}
local status=$?
if [ ${status} != 0 ]; then
log_error "Command Failed \"${@}\""
exit ${status}
fi
}
# Prepares ~/.pypirc
lock_pypirc() {
local lockfile=/var/tmp/pypirc_lock
local rc=${HOME}/.pypirc
local maxtries=10
local try=0
local sleeptime=30 #seconds
while true; do
if [[ ${try} -lt ${maxtries} ]]; then
((try++))
if ( set -o noclobber; echo "$$" > "${lockfile}") 2> /dev/null; then
log_info "Successfully acquired ${lockfile}"
echo $$ > ${lockfile}
log_info "trapping on ${lockfile}..."
trap 'rm -f "${lockfile}"; exit $?' INT TERM EXIT
# start: protected code
log_info "Creating ${rc}..."
if [ -e ${rc} ]; then
run_cmd rm -f ${rc}
fi
cat <<EOT >> ${rc}
[distutils]
index-servers =
production
staging
[production]
repository: ${PYPISERVER}
username: ${PYPIUSER}
password: ${PYPIPASS}
[staging]
repository: ${TESTSERVER}
username: ${PYPIUSER}
password: ${PYPIPASS}
EOT
run_cmd chmod 600 ${rc}
# end: protected code
break
else
log_warn "${lockfile} exists, owned by process $(cat $lockfile)"
log_info "Will retry after a ${sleeptime} seconds sleep (${try}/${maxtries})"
run_cmd sleep ${sleeptime}
fi
else
log_error "I already retried deploying ${try} times. Aborting..."
log_error "You can retry this step when less packages are building."
exit 1
fi
done
}
# Cleans ~/.pypirc, if the lock file belongs to us
unlock_pypirc() {
local lockfile=/var/tmp/pypirc_lock
local rc=${HOME}/.pypirc
# untrap if lock belongs to the running process
if [[ $(cat ${lockfile}) == $$ ]]; then
run_cmd rm -r ${lockfile}
run_cmd rm -rf ${rc}
log_info "$ trap - INT TERM EXIT"
trap - INT TERM EXIT
fi
}
# Runs setup.py in a deployment context. If fails, tries to unlock
# the ${HOME}/.pypirc file lock
setup_deploy() {
log_info "$ ${@}"
${PREFIX}/bin/python setup.py ${@}
local status=$?
if [ ${status} != 0 ]; then
log_error "Command Failed \"${@}\""
unlock_pypirc #just tries
exit ${status}
fi
}
# Uploads a file to our intranet location via curl
doc_upload() {
log_info "curl: ${1} -> ${DOCSERVER}/${2}..."
curl --location --silent --show-error --user "${DOCUSER}:${DOCPASS}" --upload-file ${1} ${DOCSERVER}/${2}
local status=$?
if [ ${status} != 0 ]; then
log_error "Curl command finished with an error condition (status=${status})"
exit ${status}
fi
log_info "Successfully uploaded ${1} with curl"
}
check_env PYTHON_VERSION
check_env CI_PROJECT_DIR
check_env CI_PROJECT_PATH
check_env CI_PROJECT_NAME
check_env CI_BUILD_REF
check_env CI_BUILD_REF_NAME
check_pass PYPIUSER
check_pass PYPIPASS
check_pass DOCUSER
check_pass DOCPASS
# Sets up variables
ARCH=`arch`
if [ -z "${CONDA_FOLDER}" ]; then
if [ "${ARCH}" == "linux" ]; then
CONDA_FOLDER=/local/conda
else
CONDA_FOLDER=/opt/conda
fi
fi
if [ -z "${CONDA_PREFIX}" ]; then
CONDA_PREFIX=env
fi
if [ -z "${DOCSERVER}" ]; then
DOCSERVER=https://www.idiap.ch
fi
PREFIX=${CI_PROJECT_DIR}/${CONDA_PREFIX}
# Add "-coverage" support
if [ "${ARCH}" == "linux" ]; then
# Temporary hack to get building done right with gcc-5 compilers
if [ -z "${CFLAGS}" ]; then
CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
else
CFLAGS="${CFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
fi
CXXFLAGS=${CFLAGS}
else
if [ -z "${CFLAGS}" ]; then
CFLAGS="-coverage"
else
CFLAGS="${CFLAGS} -coverage"
fi
CXXFLAGS=${CFLAGS}
fi
PYVER=py$(echo ${PYTHON_VERSION} | tr -d '.')
BOB_PREFIX_PATH=${CONDA_FOLDER}/envs/bob-devel-${PYVER}
LD_LIBRARY_PATH=${BOB_PREFIX_PATH}/lib
if [ -z "${PYPISERVER}" ]; then
PYPISERVER="https://pypi.python.org/pypi"
fi
TESTSERVER=https://testpypi.python.org/pypi
check_env ARCH
check_env PYVER
check_env PREFIX
export_env CFLAGS
export_env CXXFLAGS
check_env DOCSERVER
check_env PYPISERVER
check_env TESTSERVER
check_env CONDA_FOLDER
check_env CONDA_PREFIX
export_env LD_LIBRARY_PATH
export_env BOB_PREFIX_PATH
if [ -z "${CI_BUILD_TAG}" ]; then
BOB_DOCUMENTATION_SERVER="${DOCSERVER}/software/bob/docs/latest/bob/%s/master/"
export_env BOB_DOCUMENTATION_SERVER
else
log_info "Building tag, not setting BOB_DOCUMENTATION_SERVER"
fi
# taken from here: https://stackoverflow.com/questions/3685970/check-if-an-array-contains-a-value
contains_element () {
local e
for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
return 1
}
_ci/install.sh deleted 100755 → 0
+ 0
− 60
View file @ 67e58a06
#!/usr/bin/env bash
# Thu 22 Sep 2016 13:05:54 CEST
# Installation script for our build tools
if [ "${#}" -ne 1 ]; then
echo "usage: ${0} <ci-support-directory>"
echo "example: ${0} _ci"
exit 1
fi
# Functions for coloring echo commands
log_info() {
echo -e "(`date +%T`) \033[1;34m${@}\033[0m"
}
log_error() {
echo -e "(`date +%T`) \033[1;31mError: ${@}\033[0m"
}
# Function for running command and echoing results
run_cmd() {
log_info "$ ${@}"
${@}
local status=$?
if [ ${status} != 0 ]; then
log_error "Command Failed \"${@}\""
exit ${status}
fi
}
get_script() {
local url="https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/${2}"
local curlopt="--location --silent --show-error --output ${1}/${2}"
if [ -e ${1}/${2} ]; then
rm -f ${1}/${2}
fi
run_cmd curl ${curlopt} ${url}
}
get_exec() {
get_script ${1} ${2}
run_cmd chmod 755 ${1}/${2}
}
run_cmd mkdir -pv ${1}
get_script ${1} functions.sh
get_exec ${1} install.sh
for stage in "build" "test" "docs" "wheels" "deploy"; do
get_exec ${1} before_${stage}.sh
get_exec ${1} ${stage}.sh
get_exec ${1} after_${stage}.sh
done
get_exec ${1} update_feedstock.py
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment