Clean-up and integrate to the new CI

.gitignore

@@ -13,3 +13,4 @@ sphinx
 dist
 conda
 *.sql3
+record.txt

.gitlab-ci.yml

-# This build file heavily uses template features from YAML so it is generic
-# enough for any Bob project. Don't modify it unless you know what you're
-# doing.
+# This build file uses template features from YAML so it is generic enough for
+# any Bob project. Don't modify it unless you know what you're doing.
 
+# Definition of global variables (all stages)
+variables:
+  CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
 
-# Definition of our build pipeline
+
+# Definition of our build pipeline order
 stages:
   - build
-  - test
-  - docs
-  - wheels
   - deploy
+  - pypi
 
 
-# ---------
-# Templates
-# ---------
-
-# Template for the build stage
-# Needs to run on all supported architectures, platforms and python versions
+# Build targets
 .build_template: &build_job
   stage: build
   before_script:
-    - git clean -ffdx
     - mkdir _ci
     - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
     - chmod 755 _ci/install.sh
-    - ./_ci/install.sh _ci #updates
+    - ./_ci/install.sh _ci master #installs ci support scripts
     - ./_ci/before_build.sh
   script:
     - ./_ci/build.sh
   after_script:
     - ./_ci/after_build.sh
-  artifacts:
-    expire_in: 1 week
+  cache: &build_caches
     paths:
-      - _ci/
-      - dist/
-      - sphinx/
-
-
-# Template for the test stage - re-installs from uploaded wheels
-# Needs to run on all supported architectures, platforms and python versions
-.test_template: &test_job
-  stage: test
-  before_script:
-    - ./_ci/install.sh _ci #updates
-    - ./_ci/before_test.sh
-  script:
-    - ./_ci/test.sh
-  after_script:
-    - ./_ci/after_test.sh
-
-
-# Template for the wheel uploading stage
-# Needs to run against one supported architecture, platform and python version
-.wheels_template: &wheels_job
-  stage: wheels
-  environment: intranet
-  only:
-    - master
-    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
-  before_script:
-    - ./_ci/install.sh _ci #updates
-    - ./_ci/before_wheels.sh
-  script:
-    - ./_ci/wheels.sh
-  after_script:
-    - ./_ci/after_wheels.sh
-
-
-# Template for (latest) documentation upload stage
-# Only one real job needs to do this
-.docs_template: &docs_job
-  stage: docs
-  environment: intranet
-  only:
-    - master
-  before_script:
-    - ./_ci/install.sh _ci #updates
-    - ./_ci/before_docs.sh
-  script:
-    - ./_ci/docs.sh
-  after_script:
-    - ./_ci/after_docs.sh
-
-
-# Template for the deployment stage - re-installs from uploaded wheels
-# Needs to run on a single architecture only
-# Will deploy your package to PyPI and other required services
-# Only runs for tags
-.deploy_template: &deploy_job
-  stage: deploy
-  environment: internet
-  only:
-    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
-  except:
-    - branches
-  before_script:
-    - ./_ci/install.sh _ci #updates
-    - ./_ci/before_deploy.sh
-  script:
-    - ./_ci/deploy.sh
-  after_script:
-    - ./_ci/after_deploy.sh
-
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
 
-# -------------
-# Build Targets
-# -------------
 
-# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
-build_linux_27:
+.build_linux_template: &linux_build_job
   <<: *build_job
-  variables: &linux_27_build_variables
-    PYTHON_VERSION: "2.7"
-    WHEEL_TAG: "py27"
-  tags:
-    - conda-linux
-
-test_linux_27:
-  <<: *test_job
-  variables: *linux_27_build_variables
-  dependencies:
-    - build_linux_27
-  tags:
-    - conda-linux
-
-wheels_linux_27:
-  <<: *wheels_job
-  variables: *linux_27_build_variables
-  dependencies:
-    - build_linux_27
-  tags:
-    - conda-linux
-
-deploy_linux_27:
-  <<: *deploy_job
-  variables: *linux_27_build_variables
-  dependencies:
-    - build_linux_27
   tags:
-    - conda-linux
+    - docker
+  image: continuumio/conda-concourse-ci
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
+  cache:
+    <<: *build_caches
+    key: "linux-cache"
 
 
-# Linux + Python 3.5: Builds, tests and uploads wheel
-build_linux_35:
+.build_macosx_template: &macosx_build_job
   <<: *build_job
-  variables: &linux_35_build_variables
-    PYTHON_VERSION: "3.5"
-    WHEEL_TAG: "py3"
-  tags:
-    - conda-linux
-
-test_linux_35:
-  <<: *test_job
-  variables: *linux_35_build_variables
-  dependencies:
-    - build_linux_35
   tags:
-    - conda-linux
+    - macosx
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
+  cache:
+    <<: *build_caches
+    key: "macosx-cache"
 
-wheels_linux_35:
-  <<: *wheels_job
-  variables: *linux_35_build_variables
-  dependencies:
-    - build_linux_35
-  tags:
-    - conda-linux
 
-docs_linux_35:
-  <<: *docs_job
-  variables: *linux_35_build_variables
-  dependencies:
-    - build_linux_35
-  tags:
-    - conda-linux
+build_linux_27:
+  <<: *linux_build_job
+  variables:
+    PYTHON_VERSION: "2.7"
 
 
-# Linux + Python 3.6: Builds and tests
 build_linux_36:
-  <<: *build_job
-  variables: &linux_36_build_variables
+  <<: *linux_build_job
+  variables:
     PYTHON_VERSION: "3.6"
-    WHEEL_TAG: "py3"
-  tags:
-    - conda-linux
-
-test_linux_36:
-  <<: *test_job
-  variables: *linux_36_build_variables
-  dependencies:
-    - build_linux_36
-  tags:
-    - conda-linux
+    BUILD_EGG: "true"
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - dist/*.zip
+      - sphinx
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
 
 
-# Mac OSX + Python 2.7: Builds and tests
 build_macosx_27:
-  <<: *build_job
-  variables: &macosx_27_build_variables
+  <<: *macosx_build_job
+  variables:
     PYTHON_VERSION: "2.7"
-    WHEEL_TAG: "py27"
-  tags:
-    - conda-macosx
 
-test_macosx_27:
-  <<: *test_job
-  variables: *macosx_27_build_variables
+
+build_macosx_36:
+  <<: *macosx_build_job
+  variables:
+    PYTHON_VERSION: "3.6"
+
+
+# Deploy targets
+.deploy_template: &deploy_job
+  stage: deploy
+  before_script:
+    - ./_ci/install.sh _ci master #updates ci support scripts
+  script:
+    - ./_ci/deploy.sh
   dependencies:
+    - build_linux_27
+    - build_linux_36
     - build_macosx_27
+    - build_macosx_36
   tags:
-    - conda-macosx
+    - deployer
 
 
-# Mac OSX + Python 3.5: Builds and tests
-build_macosx_35:
-  <<: *build_job
-  variables: &macosx_35_build_variables
-    PYTHON_VERSION: "3.5"
-    WHEEL_TAG: "py3"
-  tags:
-    - conda-macosx
+deploy_beta:
+  <<: *deploy_job
+  environment: beta
+  only:
+    - master
 
-test_macosx_35:
-  <<: *test_job
-  variables: *macosx_35_build_variables
-  dependencies:
-    - build_macosx_35
-  tags:
-    - conda-macosx
 
+deploy_stable:
+  <<: *deploy_job
+  environment: stable
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
 
-# Mac OSX + Python 3.6: Builds and tests
-build_macosx_36:
-  <<: *build_job
-  variables: &macosx_36_build_variables
-    PYTHON_VERSION: "3.6"
-    WHEEL_TAG: "py3"
-  tags:
-    - conda-macosx
 
-test_macosx_36:
-  <<: *test_job
-  variables: *macosx_36_build_variables
+pypi:
+  stage: pypi
+  environment: pypi
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+  before_script:
+    - ./_ci/install.sh _ci master #updates ci support scripts
+  script:
+    - ./_ci/pypi.sh
   dependencies:
-    - build_macosx_36
+    - build_linux_36
   tags:
-    - conda-macosx
+    - deployer

bob/db/swan/test.py

@@ -5,12 +5,13 @@
 """
 
 from .query_bio import Database
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 def _test_numbers(files, n_total_files, n_clients, n_recordings,
                   n_devices, n_sessions, session_list, sites):
-    assert len(files) == n_total_files, len(files)
-
     n_clients_ = len(set(f.client_id for f in files))
     assert n_clients_ == n_clients, n_clients_
 
@@ -29,36 +30,50 @@ def _test_numbers(files, n_total_files, n_clients, n_recordings,
     sites_ = set(f.client.institute for f in files)
     assert sites_ == set(sites), sites_
 
+    assert len(files) == n_total_files, len(files)
+
+
+def _test_annotation(db, files):
+    try:
+        annot = db.annotations(files[0])
+        assert annot is None or isinstance(annot, dict), type(annot)
+    except AssertionError:
+        raise
+    except Exception:
+        logger.warn(
+            "annotations tests failed. Maybe the annotations files are "
+            "missing?", exc_info=True)
+
 
 def test_idiap0_voice():
-    protocol = 'idiap0-voice'
+    protocol = 'idiap0-voice-bio'
     db = Database(protocol=protocol)
 
     files = db.objects(protocol=protocol, groups='world')
     # 20 clients, 8 recordings, (2 devices in session 1 and 1 device in
     # sessions 2-6) == like it is 1 device and 7 sessions
     _test_numbers(files, 20 * 8 * 1 * 7, 20, 8, 2, 6, range(1, 7), ['IDIAP'])
-    assert all(f.client.id_in_site < 25 for f in files)
+    assert all(int(f.client.id_in_site) < 25 for f in files)
 
     files = db.objects(protocol=protocol, groups='dev', purposes='enroll')
     _test_numbers(files, 15 * 8 * 2 * 1, 15, 8, 2, 1, range(1, 2), ['IDIAP'])
-    assert all(f.client.id_in_site >=
-               25 and f.client.id_in_site < 41 for f in files)
+    assert all(int(f.client.id_in_site) >=
+               25 and int(f.client.id_in_site) < 41 for f in files)
 
     files = db.objects(protocol=protocol, groups='dev', purposes='probe')
     _test_numbers(files, 15 * 8 * 1 * 5, 15, 8, 1, 5, range(2, 7), ['IDIAP'])
-    assert all(f.client.id_in_site >=
-               25 and f.client.id_in_site < 41 for f in files)
+    assert all(int(f.client.id_in_site) >=
+               25 and int(f.client.id_in_site) < 41 for f in files)
 
     files = db.objects(protocol=protocol, groups='eval', purposes='enroll')
     _test_numbers(files, 15 * 8 * 2 * 1, 15, 8, 2, 1, range(1, 2), ['IDIAP'])
-    assert all(f.client.id_in_site >=
-               41 and f.client.id_in_site < 61 for f in files)
+    assert all(int(f.client.id_in_site) >=
+               41 and int(f.client.id_in_site) < 61 for f in files)
 
     files = db.objects(protocol=protocol, groups='eval', purposes='probe')
     _test_numbers(files, 15 * 8 * 1 * 5, 15, 8, 1, 5, range(2, 7), ['IDIAP'])
-    assert all(f.client.id_in_site >=
-               41 and f.client.id_in_site < 61 for f in files)
+    assert all(int(f.client.id_in_site) >=
+               41 and int(f.client.id_in_site) < 61 for f in files)
 
     model_ids = db.model_ids_with_protocol(groups='world', protocol=protocol)
     assert len(model_ids) == 20, len(model_ids)
@@ -67,33 +82,32 @@ def test_idiap0_voice():
     model_ids = db.model_ids_with_protocol(groups='eval', protocol=protocol)
     assert len(model_ids) == 15, len(model_ids)
 
-    assert db.annotations(files[0]) is None
+    _test_annotation(db, files)
 
 
 def test_grandtest0_voice():
-    protocol = 'grandtest0-voice'
+    protocol = 'grandtest0-voice-bio'
     db = Database(protocol=protocol)
 
     files = db.objects(protocol=protocol, groups='dev', purposes='enroll')
-    # 53 not 56 in total since MPH-FRA does not have tablet recordings
-    _test_numbers(files, 53 * 8 * 2 * 1, 56, 8, 2, 1,
-                  range(1, 2), ['IDIAP', 'MPH-FRA'])
+    _test_numbers(files, 56 * 8 * 1 * 1, 56, 8, 1, 1,
+                  [2], ['IDIAP', 'MPH-FRA'])
 
     files = db.objects(protocol=protocol, groups='dev', purposes='probe')
-    _test_numbers(files, 56 * 8 * 1 * 5, 56, 8, 1, 5,
-                  range(2, 7), ['IDIAP', 'MPH-FRA'])
+    _test_numbers(files, 2640, 56, 8, 2, 5,
+                  [1, 3, 4, 5, 6], ['IDIAP', 'MPH-FRA'])
 
     files = db.objects(protocol=protocol, groups='eval', purposes='enroll')
-    _test_numbers(files, 95 * 8 * 2 * 1, 95, 8, 2, 1,
-                  range(1, 2), ['MPH-IND', 'NTNU'])
+    _test_numbers(files, 94 * 8 * 1 * 1, 94, 8, 1, 1,
+                  [2], ['MPH-IND', 'NTNU'])
 
     files = db.objects(protocol=protocol, groups='eval', purposes='probe')
-    _test_numbers(files, 95 * 8 * 1 * 5, 95, 8, 1, 5,
-                  range(2, 7), ['MPH-IND', 'NTNU'])
+    _test_numbers(files, 4512, 94, 8, 2, 5,
+                  [1, 3, 4, 5, 6], ['MPH-IND', 'NTNU'])
 
     model_ids = db.model_ids_with_protocol(groups='dev', protocol=protocol)
     assert len(model_ids) == 56, len(model_ids)
     model_ids = db.model_ids_with_protocol(groups='eval', protocol=protocol)
-    assert len(model_ids) == 95, len(model_ids)
+    assert len(model_ids) == 94, len(model_ids)
 
-    assert db.annotations(files[0]) is None
+    _test_annotation(db, files)

doc/index.rst

@@ -6,22 +6,20 @@
  SWAN Database Access API for Bob
 ==================================
 
-To use this database, you may need to download additional files:
+This package provides an API to the protocols of the SWAN database.
 
-.. code-block:: sh
-
-	$ bob_dbmanage.py swan download --missing
-	
-To configure the location of the database on your computer:
+To configure the location of the database and the location of face annotations
+on your computer:
 
 .. code-block:: sh
 
     $ bob config set bob.db.swan.directory /path/to/swan/database
+    $ bob config set bob.db.swan.annotation_dir /path/to/swan/annotations
 
 
 Package Documentation
 ---------------------
 
 .. automodule:: bob.db.swan
-.. automodule:: bob.db.swan.query
-.. automodule:: bob.db.swan.models
+.. automodule:: bob.db.swan.query_bio
+.. automodule:: bob.db.swan.query_pad

requirements.txt

@@ -8,3 +8,6 @@ bob.bio.base
 bob.bio.spear
 bob.bio.face
 bob.bio.video
+bob.pad.base
+bob.pad.face
+bob.pad.voice