Commit a66b52b7 authored by André Anjos's avatar André Anjos 💬

Merge branch 'new-ci' into 'master'

New CI

See merge request !263
parents 6f223292 aada9a3e
Pipeline #25495 passed with stages
in 17 minutes and 3 seconds
......@@ -27,3 +27,6 @@ html/
*.tar.bz2
nohup.out
*.log
_ci/
scripts/_core_docker_pull.sh
.beatrc
stages:
- build
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
DOCKER_REGISTRY: docker.idiap.ch
TEST_PREFIX_PATH_FILE: 'test_prefix_$CI_JOB_ID.txt'
CONDA_ENV_NAME: beat_test_env
CONDA_PREFIX: /local/opt/conda
PREFIX: $CONDA_PREFIX/envs/$CONDA_ENV_NAME
build:
# Definition of our build pipeline order
stages:
- build
- deploy
- pypi
# Build targets
.build_template: &build_job
stage: build
before_script:
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN docker.idiap.ch
- export BEAT_TEST_PREFIX=`mktemp -d --tmpdir=/var/tmp beat_test_prefix.XXXXXXXXX`
- echo $BEAT_TEST_PREFIX > $TEST_PREFIX_PATH_FILE
- ./scripts/before_build.sh
script:
- ./scripts/build.sh
after_script:
- ./scripts/after_build.sh
- ./_ci/after_build.sh
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker-build
cache:
<<: *build_caches
key: "linux-cache"
build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
except:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ${PREFIX}/bin/python --version
- docker info
- git clean -ffdx
- export BEAT_TEST_PREFIX=`mktemp -d --tmpdir=/var/tmp beat_test_prefix.XXXXXXXXX`
- echo $BEAT_TEST_PREFIX > $TEST_PREFIX_PATH_FILE
- export PATH="$CONDA_PREFIX/bin:$PATH"
- source activate $CONDA_ENV_NAME
- conda info --envs
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ${PREFIX}/bin/python bootstrap-buildout.py
- ./bin/buildout
- export COVERAGE_FILE=.coverage.django
- ./bin/python ${PREFIX}/bin/coverage run --source=${CI_PROJECT_NAME} ./bin/django test --settings=beat.web.settings.ci -v 2
- export BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.ci
- export COVERAGE_FILE=.coverage.cmdline
- export NOSE_WITH_COVERAGE=1
- export NOSE_COVER_PACKAGE=beat.web
- ./bin/python ${PREFIX}/bin/coverage run --source=./src/beat.cmdline ./bin/nosetests -sv beat.cmdline
- unset COVERAGE_FILE
- ./bin/python ${PREFIX}/bin/coverage combine .coverage.django .coverage.cmdline
- ./bin/python ${PREFIX}/bin/coverage report
- ./bin/python ${PREFIX}/bin/sphinx-apidoc --separate -d 2 --output=doc/api ${CI_PROJECT_NAMESPACE} beat/web/*/migrations beat/web/*/tests
- ./bin/python ${PREFIX}/bin/sphinx-build doc/api html/api
- ./bin/python ${PREFIX}/bin/sphinx-build doc/admin html/admin
- ./bin/python ${PREFIX}/bin/sphinx-build doc/user html/user
after_script:
- export BEAT_TEST_PREFIX=`cat $TEST_PREFIX_PATH_FILE`
- echo "Deleting BEAT test prefix $BEAT_TEST_PREFIX"
- rm -rf $BEAT_TEST_PREFIX $TEST_PREFIX_PATH_FILE
- unset BEAT_TEST_PREFIX
- ./_ci/deploy.sh
dependencies:
- build_linux_36
tags:
- deployer
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
pypi:
stage: pypi
environment: pypi
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/pypi.sh
dependencies:
- build_linux_36
tags:
- docker
- deployer
include LICENSE.AGPL README.rst bootstrap-buildout.py buildout.cfg
include LICENSE.AGPL README.rst buildout.cfg common.cfg release.cfg
recursive-include beat/web *.css *.png *.gif *.svg *.ico *.js *.html *.rst *.txt
recursive-include doc conf.py *.rst *.png *.svg *.ico *.pdf
recursive-include beat/web/backend/environments/default/bin execute describe
......@@ -38,23 +38,14 @@ Installation
Here is a recipe to get you started::
$ python bootstrap-buildout.py
$ ./bin/buildout
$ conda env create -f dev.yml
$ conda activate beat-web
$ buildout
$ ./scripts/docker_pull.sh master
These 2 commands should download and install all non-installed dependencies and
These commands should download and install all non-installed dependencies and
get you a fully operational test and development environment.
.. note::
The python shell used in the first line of the previous command set
determines the python interpreter that will be used for all scripts developed
inside this package.
If you are on the Idiap filesystem, you may use
``/idiap/project/beat/environments/staging/usr/bin/python`` to bootstrap this
package instead. It contains the same setup deployed at the final BEAT
machinery.
Documentation
-------------
......
##############################################################################
#
# Copyright (c) 2006 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Bootstrap a buildout-based project
Simply run this script in a directory containing a buildout.cfg.
The script accepts buildout command-line options, so you can
use the -c option to specify an alternate configuration file.
"""
import os
import shutil
import sys
import tempfile
from optparse import OptionParser
tmpeggs = tempfile.mkdtemp()
usage = '''\
[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
Bootstraps a buildout-based project.
Simply run this script in a directory containing a buildout.cfg, using the
Python that you want bin/buildout to use.
Note that by using --find-links to point to local resources, you can keep
this script from going over the network.
'''
parser = OptionParser(usage=usage)
parser.add_option("-v", "--version", help="use a specific zc.buildout version")
parser.add_option("-t", "--accept-buildout-test-releases",
dest='accept_buildout_test_releases',
action="store_true", default=False,
help=("Normally, if you do not specify a --version, the "
"bootstrap script and buildout gets the newest "
"*final* versions of zc.buildout and its recipes and "
"extensions for you. If you use this flag, "
"bootstrap and buildout will get the newest releases "
"even if they are alphas or betas."))
parser.add_option("-c", "--config-file",
help=("Specify the path to the buildout configuration "
"file to be used."))
parser.add_option("-f", "--find-links",
help=("Specify a URL to search for buildout releases"))
parser.add_option("--allow-site-packages",
action="store_true", default=False,
help=("Let bootstrap.py use existing site packages"))
parser.add_option("--setuptools-version",
help="use a specific setuptools version")
options, args = parser.parse_args()
######################################################################
# load/install setuptools
try:
if options.allow_site_packages:
import setuptools
import pkg_resources
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
ez = {}
exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
if not options.allow_site_packages:
# ez_setup imports site, which adds site packages
# this will remove them from the path to ensure that incompatible versions
# of setuptools are not in the path
import site
# inside a virtualenv, there is no 'getsitepackages'.
# We can't remove these reliably
if hasattr(site, 'getsitepackages'):
for sitepackage_path in site.getsitepackages():
sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
setup_args = dict(to_dir=tmpeggs, download_delay=0)
if options.setuptools_version is not None:
setup_args['version'] = options.setuptools_version
ez['use_setuptools'](**setup_args)
import setuptools
import pkg_resources
# This does not (always?) update the default working set. We will
# do it.
for path in sys.path:
if path not in pkg_resources.working_set.entries:
pkg_resources.working_set.add_entry(path)
######################################################################
# Install buildout
ws = pkg_resources.working_set
cmd = [sys.executable, '-c',
'from setuptools.command.easy_install import main; main()',
'-mZqNxd', tmpeggs]
find_links = os.environ.get(
'bootstrap-testing-find-links',
options.find_links or
('http://downloads.buildout.org/'
if options.accept_buildout_test_releases else None)
)
if find_links:
cmd.extend(['-f', find_links])
setuptools_path = ws.find(
pkg_resources.Requirement.parse('setuptools')).location
requirement = 'zc.buildout'
version = options.version
if version is None and not options.accept_buildout_test_releases:
# Figure out the most recent final version of zc.buildout.
import setuptools.package_index
_final_parts = '*final-', '*final'
def _final_version(parsed_version):
try:
return not parsed_version.is_prerelease
except AttributeError:
# Older setuptools
for part in parsed_version:
if (part[:1] == '*') and (part not in _final_parts):
return False
return True
index = setuptools.package_index.PackageIndex(
search_path=[setuptools_path])
if find_links:
index.add_find_links((find_links,))
req = pkg_resources.Requirement.parse(requirement)
if index.obtain(req) is not None:
best = []
bestv = None
for dist in index[req.project_name]:
distv = dist.parsed_version
if _final_version(distv):
if bestv is None or distv > bestv:
best = [dist]
bestv = distv
elif distv == bestv:
best.append(dist)
if best:
best.sort()
version = best[-1].version
if version:
requirement = '=='.join((requirement, version))
cmd.append(requirement)
import subprocess
if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
raise Exception(
"Failed to execute command:\n%s" % repr(cmd)[1:-1])
######################################################################
# Import and run buildout
ws.add_entry(tmpeggs)
ws.require(requirement)
import zc.buildout.buildout
if not [a for a in args if '=' not in a]:
args.append('bootstrap')
# if -c was provided, we push it back into args for buildout' main function
if options.config_file is not None:
args[0:0] = ['-c', options.config_file]
zc.buildout.buildout.main(args)
shutil.rmtree(tmpeggs)
[buildout]
extends = common.cfg
parts += docker_images cxx_algorithms
index = https://pypi.org/simple
eggs += ipdb
coverage
pip
autopep8
develop = .
[sources]
beat.core = git git@gitlab.idiap.ch:beat/beat.core
beat.cmdline = git git@gitlab.idiap.ch:beat/beat.cmdline branch=1.4.x
beat.backend.python = git git@gitlab.idiap.ch:beat/beat.backend.python
beat.examples = git git@gitlab.idiap.ch:beat/beat.examples egg=false
[docker_images]
recipe = collective.recipe.cmd
cmds = ./src/beat.core/buildout_pull_images.sh
uninstall_cmds =
on_install = true
on_update = true
[cxx_algorithms]
recipe = collective.recipe.cmd
cmds = ./src/beat.core/buildout_compile_cxx_algorithm.sh build
uninstall_cmds = ./src/beat.core/buildout_compile_cxx_algorithm.sh cleanup
on_install = true
on_update = true
[node]
npms += protractor
scripts += protractor webdriver-manager
[bower]
base-directory = beat/web
\ No newline at end of file
base-directory = beat/web
[buildout]
parts = syseggs scripts django node bower help
parts = scripts node bower help django
extensions = mr.developer
auto-checkout = *
develop = .
newest = false
eggs = beat.web
beat.cmdline
beat.core
beat.backend.python
versions = versions
[scripts]
recipe = bob.buildout:scripts
......@@ -19,7 +17,6 @@ recipe = djangorecipe
project = beat
settings = web.settings.settings
eggs = ${buildout:eggs}
django
[node]
recipe = gp.recipe.node
......@@ -55,83 +52,9 @@ packages = jquery#~1.11.3
executable = ${buildout:bin-directory}/bower --config.interactive=false
downloads = static
[versions]
django = >=1.11,<2.0
django-rest-swagger = >2.1
django-guardian = >=1.3
djangorestframework = >3.7
django-activity-stream = >= 0.6.5
django-jsonfield = >= 1.0.1
[syseggs]
recipe = syseggrecipe
;force-sysegg = true
eggs = alabaster
babel
backports.shutil-get-terminal-size
click
click-plugins
colorlog
coverage
cycler
decorator
django
django-activity-stream
django-guardian
djangorestframework
django-rest-swagger
django-jsonfield
django-jsonfield-compat
docopt
docutils
funcsigs
functools32
graphviz
ipdb
ipython
ipython-genutils
imagesize
Jinja2
jsonschema
MarkupSafe
matplotlib
mock
numpy
oset
pathlib2
pbr
pexpect
pickleshare
pillow
pip
psutil
ptyprocess
psycopg2-binary
Pygments
pyparsing
python-archive
python-dateutil
pytz
PyYAML
pyzmq
simplegeneric
simplejson
six
snowballstemmer
Sphinx
sphinxcontrib-ansi
sphinxcontrib-httpdomain
sphinxcontrib-mscgen
sphinxcontrib-programoutput
sphinx-rtd-theme
termcolor
traitlets
uwsgi
[help]
base-path = .
recipe = collective.recipe.cmd
cmds = ${buildout:bin-directory}/sphinx-build ${help:base-path}/doc/user ${help:base-path}/beat/web/static/guide
uninstall_cmds = rm -rf ${help:base-path}/beat/web/static/guide
on_install = true
on_update = true
\ No newline at end of file
on_update = true
name: beat-web
channels:
- http://www.idiap.ch/software/beat/conda/label/beta
- http://www.idiap.ch/software/beat/conda
- defaults
dependencies:
# packages you may pin are here
- python=3.6
- bob-devel=2018.12.11
- beat-devel=2018.12.12
# requirements.txt, they are indirectly pinned through the above
- beat.core
- docopt
- docutils
- jinja2
- matplotlib
- psutil
- psycopg2
- pytz
- simplejson
# these are required for local development, they are not runtime
- beat.cmdline
- bob.extension
- nose
- coverage
- sphinx
- sphinx_rtd_theme
- bob.buildout
- mr.developer
- sphinxcontrib-programoutput
- sphinxcontrib-httpdomain
# add here, packages which are **not** available on above channels
- pip:
- django>=1.11,<2.0
- django-activity-stream>=0.6,<0.7
- django-guardian>=1.4,<1.5
- django-jsonfield>=1.0,<1.1
- django-jsonfield-compat>=0.4,<0.5
- django-post_office>=3.1,<3.2
- django-rest-swagger>=2.2,<2.3
- djangorestframework>=3.9,<3.10
......@@ -81,7 +81,7 @@ platform, in case you don't manage it from the provided instructions.
I'm getting an error, or the platform is not functioning as expected, or the platform does not do something I need. How can I submit a feature request or a bug report?
----------------------------------------------------------------------------------------------
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
It is great you asked! Always start by posting your question at our
`development mailing list`_. If the new feature or bug report is confirmed, we
......
......@@ -5,9 +5,19 @@ always-checkout = force
eggs-directory = .buildout/eggs
download-cache = .buildout/download-cache
abi-tag-eggs = true
versions = versions
develop = beat.backend.python
beat.core
beat.cmdline
.
[syseggs]
eggs += django-post_office
[versions]
django = >=1.11,<2.0
django-rest-swagger = >2.1
django-guardian = >=1.3
djangorestframework = >3.7
django-activity-stream = >= 0.6.5
django-jsonfield = >= 1.0.1
[scripts]
eggs = ${buildout:eggs}
......
#!/usr/bin/env bash
source _ci/functions.sh
BEAT_TEST_PREFIX=`cat $TEST_PREFIX_PATH_FILE`
export_env BEAT_TEST_PREFIX
log_info "Deleting BEAT test prefix $BEAT_TEST_PREFIX"
run_cmd rm -rf ${BEAT_TEST_PREFIX} ${TEST_PREFIX_PATH_FILE}
#!/usr/bin/env bash
source _ci/functions.sh
# Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate base
export_env PATH
# installs the base conda environment
run_cmd conda env create -f dev.yml
conda activate beat-web
# runs buildout
run_cmd buildout
# pull docker images
thisdir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
run_cmd ${thisdir}/docker_pull.sh master
#!/usr/bin/env bash
source _ci/functions.sh
# Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate beat-web
export_env PATH
COVERAGE_FILE=.coverage.django
export_env COVERAGE_FILE
run_cmd ./bin/coverage run --source=${CI_PROJECT_NAME} ./bin/django test --settings=beat.web.settings.ci -v 2
BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.ci
COVERAGE_FILE=.coverage.cmdline
NOSE_WITH_COVERAGE=1
NOSE_COVER_PACKAGE=beat.web
export_env BEAT_CMDLINE_TEST_PLATFORM
export_env COVERAGE_FILE
export_env NOSE_WITH_COVERAGE
export_env NOSE_COVER_PACKAGE
run_cmd ./bin/coverage run --source=beat.cmdline ./bin/nosetests -sv beat.cmdline
unset COVERAGE_FILE
run_cmd ./bin/coverage combine .coverage.django .coverage.cmdline
run_cmd ./bin/coverage report
run_cmd ./bin/sphinx-apidoc --separate -d 2 --output=doc/api ${CI_PROJECT_NAMESPACE} beat/web/*/migrations beat/web/*/tests
run_cmd ./bin/sphinx-build doc/api sphinx/api
run_cmd ./bin/sphinx-build doc/admin sphinx/admin
run_cmd ./bin/sphinx-build doc/user sphinx/user
# Builds a zip file that can be uploaded to PyPI
python setup.py sdist --formats=zip
#!/usr/bin/env bash
if [ $# == 0 ]; then
echo "usage: $0 <beat-core-branch>"
exit 1
fi
basedir=`pwd`
if [ ! -e ${basedir}/_ci/functions.sh ]; then
# this will happen when executing buildout for the first time
mkdir ${basedir}/_ci
curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > ${basedir}/_ci/install.sh
chmod 755 ${basedir}/_ci/install.sh
${basedir}/_ci/install.sh ${basedir}/_ci master #installs ci support scripts
else
${basedir}/_ci/install.sh ${basedir}/_ci master #updates ci support scripts
fi
if [ "${BUILDOUT}" == "true" ]; then
# when executing in the context of buildout, define dummies
export CI_PROJECT_URL=https://gitlab.idiap.ch/beat/beat.web
export CI_PROJECT_DIR=beat
export CI_PROJECT_PATH=beat/beat.web
export CI_PROJECT_NAME=beat.web
export CI_COMMIT_REF_NAME=master
export PYPIUSER=pypiuser
export PYPIPASS=pypipass
export DOCUSER=docuser
export DOCPASS=docpass
export CONDA_ROOT=${basedir}/miniconda
export DOCKER_REGISTRY=docker.idiap.ch
export DOCSERVER=https://www.idiap.ch/software/bob
fi
source ${basedir}/_ci/functions.sh
branch=$1
check_env branch
destdir="$(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd)"
check_env destdir
run_cmd curl -o ${destdir}/_core_docker_pull.sh --silent "https://gitlab.idiap.ch/beat/beat.core/raw/${branch}/scripts/docker_pull.sh"
run_cmd chmod 755 ${destdir}/_core_docker_pull.sh
run_cmd ${destdir}/_core_docker_pull.sh
......@@ -26,14 +26,17 @@
###############################################################################
from setuptools import setup, find_packages
from io import open
def load_requirements(f):
retval = [str(k.strip()) for k in open(f, 'rt')]
return [k for k in retval if k and k[0] not in ('#', '-')]
# The only thing we do in this file is to call the setup() function with all
# parameters that define our package.
setup(
name='beat.web',
version='1.3.1rc1',
version=open("version.txt").read().rstrip(),
description='Biometrics Evaluation and Testing Platform (Web Modules)',
url='https://gitlab.idiap.ch/beat/beat.web',
license='AGPLv3',
......@@ -45,35 +48,7 @@ setup(
packages=find_packages(),
include_package_data=True,
zip_safe=False,
namespace_packages=[
"beat",