Commit 81f5e2af authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Clean-up and integrate to the new CI

parent 9101dc3b
Pipeline #19924 failed with stages
in 4 minutes and 12 seconds
......@@ -13,3 +13,4 @@ sphinx
dist
conda
*.sql3
record.txt
# This build file heavily uses template features from YAML so it is generic
# enough for any Bob project. Don't modify it unless you know what you're
# doing.
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
# Definition of our build pipeline
# Definition of our build pipeline order
stages:
- build
- test
- docs
- wheels
- deploy
- pypi
# ---------
# Templates
# ---------
# Template for the build stage
# Needs to run on all supported architectures, platforms and python versions
# Build targets
.build_template: &build_job
stage: build
before_script:
- git clean -ffdx
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci #updates
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
artifacts:
expire_in: 1 week
cache: &build_caches
paths:
- _ci/
- dist/
- sphinx/
# Template for the test stage - re-installs from uploaded wheels
# Needs to run on all supported architectures, platforms and python versions
.test_template: &test_job
stage: test
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_test.sh
script:
- ./_ci/test.sh
after_script:
- ./_ci/after_test.sh
# Template for the wheel uploading stage
# Needs to run against one supported architecture, platform and python version
.wheels_template: &wheels_job
stage: wheels
environment: intranet
only:
- master
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_wheels.sh
script:
- ./_ci/wheels.sh
after_script:
- ./_ci/after_wheels.sh
# Template for (latest) documentation upload stage
# Only one real job needs to do this
.docs_template: &docs_job
stage: docs
environment: intranet
only:
- master
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_docs.sh
script:
- ./_ci/docs.sh
after_script:
- ./_ci/after_docs.sh
# Template for the deployment stage - re-installs from uploaded wheels
# Needs to run on a single architecture only
# Will deploy your package to PyPI and other required services
# Only runs for tags
.deploy_template: &deploy_job
stage: deploy
environment: internet
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_deploy.sh
script:
- ./_ci/deploy.sh
after_script:
- ./_ci/after_deploy.sh
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
# -------------
# Build Targets
# -------------
# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
build_linux_27:
.build_linux_template: &linux_build_job
<<: *build_job
variables: &linux_27_build_variables
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-linux
test_linux_27:
<<: *test_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
wheels_linux_27:
<<: *wheels_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
deploy_linux_27:
<<: *deploy_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
- docker
image: continuumio/conda-concourse-ci
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
<<: *build_caches
key: "linux-cache"
# Linux + Python 3.5: Builds, tests and uploads wheel
build_linux_35:
.build_macosx_template: &macosx_build_job
<<: *build_job
variables: &linux_35_build_variables
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_35:
<<: *test_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
- macosx
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
cache:
<<: *build_caches
key: "macosx-cache"
wheels_linux_35:
<<: *wheels_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
docs_linux_35:
<<: *docs_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
- conda-linux
build_linux_27:
<<: *linux_build_job
variables:
PYTHON_VERSION: "2.7"
# Linux + Python 3.6: Builds and tests
build_linux_36:
<<: *build_job
variables: &linux_36_build_variables
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
WHEEL_TAG: "py3"
tags:
- conda-linux
test_linux_36:
<<: *test_job
variables: *linux_36_build_variables
dependencies:
- build_linux_36
tags:
- conda-linux
BUILD_EGG: "true"
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
# Mac OSX + Python 2.7: Builds and tests
build_macosx_27:
<<: *build_job
variables: &macosx_27_build_variables
<<: *macosx_build_job
variables:
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-macosx
test_macosx_27:
<<: *test_job
variables: *macosx_27_build_variables
build_macosx_36:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.6"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/deploy.sh
dependencies:
- build_linux_27
- build_linux_36
- build_macosx_27
- build_macosx_36
tags:
- conda-macosx
- deployer
# Mac OSX + Python 3.5: Builds and tests
build_macosx_35:
<<: *build_job
variables: &macosx_35_build_variables
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-macosx
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
test_macosx_35:
<<: *test_job
variables: *macosx_35_build_variables
dependencies:
- build_macosx_35
tags:
- conda-macosx
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
# Mac OSX + Python 3.6: Builds and tests
build_macosx_36:
<<: *build_job
variables: &macosx_36_build_variables
PYTHON_VERSION: "3.6"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_36:
<<: *test_job
variables: *macosx_36_build_variables
pypi:
stage: pypi
environment: pypi
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/pypi.sh
dependencies:
- build_macosx_36
- build_linux_36
tags:
- conda-macosx
- deployer
......@@ -5,12 +5,13 @@
"""
from .query_bio import Database
import logging
logger = logging.getLogger(__name__)
def _test_numbers(files, n_total_files, n_clients, n_recordings,
n_devices, n_sessions, session_list, sites):
assert len(files) == n_total_files, len(files)
n_clients_ = len(set(f.client_id for f in files))
assert n_clients_ == n_clients, n_clients_
......@@ -29,36 +30,50 @@ def _test_numbers(files, n_total_files, n_clients, n_recordings,
sites_ = set(f.client.institute for f in files)
assert sites_ == set(sites), sites_
assert len(files) == n_total_files, len(files)
def _test_annotation(db, files):
try:
annot = db.annotations(files[0])
assert annot is None or isinstance(annot, dict), type(annot)
except AssertionError:
raise
except Exception:
logger.warn(
"annotations tests failed. Maybe the annotations files are "
"missing?", exc_info=True)
def test_idiap0_voice():
protocol = 'idiap0-voice'
protocol = 'idiap0-voice-bio'
db = Database(protocol=protocol)
files = db.objects(protocol=protocol, groups='world')
# 20 clients, 8 recordings, (2 devices in session 1 and 1 device in
# sessions 2-6) == like it is 1 device and 7 sessions
_test_numbers(files, 20 * 8 * 1 * 7, 20, 8, 2, 6, range(1, 7), ['IDIAP'])
assert all(f.client.id_in_site < 25 for f in files)
assert all(int(f.client.id_in_site) < 25 for f in files)
files = db.objects(protocol=protocol, groups='dev', purposes='enroll')
_test_numbers(files, 15 * 8 * 2 * 1, 15, 8, 2, 1, range(1, 2), ['IDIAP'])
assert all(f.client.id_in_site >=
25 and f.client.id_in_site < 41 for f in files)
assert all(int(f.client.id_in_site) >=
25 and int(f.client.id_in_site) < 41 for f in files)
files = db.objects(protocol=protocol, groups='dev', purposes='probe')
_test_numbers(files, 15 * 8 * 1 * 5, 15, 8, 1, 5, range(2, 7), ['IDIAP'])
assert all(f.client.id_in_site >=
25 and f.client.id_in_site < 41 for f in files)
assert all(int(f.client.id_in_site) >=
25 and int(f.client.id_in_site) < 41 for f in files)
files = db.objects(protocol=protocol, groups='eval', purposes='enroll')
_test_numbers(files, 15 * 8 * 2 * 1, 15, 8, 2, 1, range(1, 2), ['IDIAP'])
assert all(f.client.id_in_site >=
41 and f.client.id_in_site < 61 for f in files)
assert all(int(f.client.id_in_site) >=
41 and int(f.client.id_in_site) < 61 for f in files)
files = db.objects(protocol=protocol, groups='eval', purposes='probe')
_test_numbers(files, 15 * 8 * 1 * 5, 15, 8, 1, 5, range(2, 7), ['IDIAP'])
assert all(f.client.id_in_site >=
41 and f.client.id_in_site < 61 for f in files)
assert all(int(f.client.id_in_site) >=
41 and int(f.client.id_in_site) < 61 for f in files)
model_ids = db.model_ids_with_protocol(groups='world', protocol=protocol)
assert len(model_ids) == 20, len(model_ids)
......@@ -67,33 +82,32 @@ def test_idiap0_voice():
model_ids = db.model_ids_with_protocol(groups='eval', protocol=protocol)
assert len(model_ids) == 15, len(model_ids)
assert db.annotations(files[0]) is None
_test_annotation(db, files)
def test_grandtest0_voice():
protocol = 'grandtest0-voice'
protocol = 'grandtest0-voice-bio'
db = Database(protocol=protocol)
files = db.objects(protocol=protocol, groups='dev', purposes='enroll')
# 53 not 56 in total since MPH-FRA does not have tablet recordings
_test_numbers(files, 53 * 8 * 2 * 1, 56, 8, 2, 1,
range(1, 2), ['IDIAP', 'MPH-FRA'])
_test_numbers(files, 56 * 8 * 1 * 1, 56, 8, 1, 1,
[2], ['IDIAP', 'MPH-FRA'])
files = db.objects(protocol=protocol, groups='dev', purposes='probe')
_test_numbers(files, 56 * 8 * 1 * 5, 56, 8, 1, 5,
range(2, 7), ['IDIAP', 'MPH-FRA'])
_test_numbers(files, 2640, 56, 8, 2, 5,
[1, 3, 4, 5, 6], ['IDIAP', 'MPH-FRA'])
files = db.objects(protocol=protocol, groups='eval', purposes='enroll')
_test_numbers(files, 95 * 8 * 2 * 1, 95, 8, 2, 1,
range(1, 2), ['MPH-IND', 'NTNU'])
_test_numbers(files, 94 * 8 * 1 * 1, 94, 8, 1, 1,
[2], ['MPH-IND', 'NTNU'])
files = db.objects(protocol=protocol, groups='eval', purposes='probe')
_test_numbers(files, 95 * 8 * 1 * 5, 95, 8, 1, 5,
range(2, 7), ['MPH-IND', 'NTNU'])
_test_numbers(files, 4512, 94, 8, 2, 5,
[1, 3, 4, 5, 6], ['MPH-IND', 'NTNU'])
model_ids = db.model_ids_with_protocol(groups='dev', protocol=protocol)
assert len(model_ids) == 56, len(model_ids)
model_ids = db.model_ids_with_protocol(groups='eval', protocol=protocol)
assert len(model_ids) == 95, len(model_ids)
assert len(model_ids) == 94, len(model_ids)
assert db.annotations(files[0]) is None
_test_annotation(db, files)
......@@ -6,22 +6,20 @@
SWAN Database Access API for Bob
==================================
To use this database, you may need to download additional files:
This package provides an API to the protocols of the SWAN database.
.. code-block:: sh
$ bob_dbmanage.py swan download --missing
To configure the location of the database on your computer:
To configure the location of the database and the location of face annotations
on your computer:
.. code-block:: sh
$ bob config set bob.db.swan.directory /path/to/swan/database
$ bob config set bob.db.swan.annotation_dir /path/to/swan/annotations
Package Documentation
---------------------
.. automodule:: bob.db.swan
.. automodule:: bob.db.swan.query
.. automodule:: bob.db.swan.models
.. automodule:: bob.db.swan.query_bio
.. automodule:: bob.db.swan.query_pad
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment