From c78475790b47386e77bea7d24b8b3f25a12c498b Mon Sep 17 00:00:00 2001
From: Guillaume HEUSCH <guillaume.heusch@idiap.ch>
Date: Tue, 9 Oct 2018 14:56:57 +0200
Subject: [PATCH] implementation of new test units

---
 bob/db/fargo/test.py | 187 ++++++++++++++-----------------------------
 1 file changed, 58 insertions(+), 129 deletions(-)

diff --git a/bob/db/fargo/test.py b/bob/db/fargo/test.py
index 544f0ea..18a2e9e 100644
--- a/bob/db/fargo/test.py
+++ b/bob/db/fargo/test.py
@@ -6,141 +6,70 @@
 
 """A few checks on the protocols of the FARGO public database 
 """
-from .query import Database
-from bob.bio.base.test.test_database_implementations import check_database
+import os, sys
+import bob.db.fargo
 
-#def test_fargo():
-#    db = Database() 
-#    check_database(db, protocol='mc-rgb', groups=('dev', 'eval'))
+def db_available(test):
+  """Decorator for detecting if the database file is available"""
+  from bob.io.base.test_utils import datafile
+  from nose.plugins.skip import SkipTest
+  import functools
 
-def test_query_public_MC_RGB():
-  """
-  Test some queries for the public MC RGB protocol
-  """
-  db = Database()
-  db.objects(protocol='uo-rgb', groups ='train')
+  @functools.wraps(test)
+  def wrapper(*args, **kwargs):
+    dbfile = datafile("db.sql3", __name__, None)
+    if os.path.exists(dbfile):
+      return test(*args, **kwargs)
+    else:
+      raise SkipTest("The database file '%s' is not available; did you forget to run 'bob_dbmanage.py %s create' ?" % (dbfile, 'fargo'))
 
+  return wrapper
 
- # assert len(db.client_ids(protocol='public_MC_RGB')) == 75 # 75 client ids for world, dev and eval
- # assert len(db.client_ids(protocol='public_MC_RGB', groups='world')) == 25 # 25 client ids for world
- # assert len(db.client_ids(protocol='public_MC_RGB', groups='dev')) == 25 # 25 client ids for dev
- # assert len(db.client_ids(protocol='public_MC_RGB', groups='eval')) == 25 # 25 client ids for eval
 
- # assert len(db.model_ids_with_protocol(protocol='public_MC_RGB')) == 75 # 75 model ids for world, dev and eval
- # assert len(db.model_ids_with_protocol(protocol='public_MC_RGB', groups='world')) == 25 # 25 model ids for world
- # assert len(db.model_ids_with_protocol(protocol='public_MC_RGB', groups='dev')) == 25 # 10 model ids for dev
- # assert len(db.model_ids_with_protocol(protocol='public_MC_RGB', groups='eval')) == 25 # 10 model ids for eval
+@db_available
+def test_clients():
 
- # assert len(db.objects(protocol='public_MC_RGB', groups='world')) == 1000 # 1000 samples in the world set
+  # test whether the correct number of clients is returned
+  db = bob.db.fargo.Database()
+  assert len(db.groups()) == 3
+  assert len(db.clients()) == 75
+  assert len(db.clients(groups='world')) == 25
+  assert len(db.clients(groups='dev')) == 25
+  assert len(db.clients(groups='eval')) == 25
 
- # assert len(db.objects(protocol='public_MC_RGB', groups='dev', purposes='enroll')) == 500 # 500 samples for enrollment in the dev set (25 clients * 10 images * 2 recordings)
- # assert len(db.objects(protocol='public_MC_RGB', groups='dev', purposes='enroll', model_ids='026')) == 20 # 20 samples to enroll model '026' in the dev set
- # assert len(db.objects(protocol='public_MC_RGB', groups='dev', purposes='enroll', model_ids='071')) == 0 # 0 samples to enroll model '071' (it is an eval model)
- # assert len(db.objects(protocol='public_MC_RGB', groups='dev', purposes='probe')) == 500 # 500 samples as probes in the dev set (25 clients * 10 images * 2 recordings)
 
- # assert len(db.objects(protocol='public_MC_RGB', groups='eval', purposes='enroll')) == 500 # 500 samples for enrollment in the eval set (25 clients * 10 images * 2 recordings)
- # assert len(db.objects(protocol='public_MC_RGB', groups='eval', purposes='enroll', model_ids='058')) == 20 # 20 samples to enroll model '058' in the eval set
- # assert len(db.objects(protocol='public_MC_RGB', groups='eval', purposes='enroll', model_ids='001')) == 0 # 0 samples to enroll model '001' (it is a world model)
- # assert len(db.objects(protocol='public_MC_RGB', groups='eval', purposes='probe')) == 500 # 500 samples as probes in the eval set (25 clients * 10 images * 2 recordings) 
+@db_available
+def test_objects():
+#  # tests if the right number of File objects is returned
+  
+  db = bob.db.fargo.Database()
 
-#def test_query_public_UD_RGB():
-#  """
-#  Test some queries for the public UD RGB protocol
-#  """
-#  db = Database()
-#  
-#  assert len(db.client_ids(protocol='public_UD_RGB')) == 75 # 75 client ids for world, dev and eval
-#  assert len(db.client_ids(protocol='public_UD_RGB', groups='world')) == 25 # 25 client ids for world
-#  assert len(db.client_ids(protocol='public_UD_RGB', groups='dev')) == 25 # 25 client ids for dev
-#  assert len(db.client_ids(protocol='public_UD_RGB', groups='eval')) == 25 # 25 client ids for eval
-#
-#  assert len(db.model_ids_with_protocol(protocol='public_UD_RGB')) == 75 # 75 model ids for world, dev and eval
-#  assert len(db.model_ids_with_protocol(protocol='public_UD_RGB', groups='world')) == 25 # 25 model ids for world
-#  assert len(db.model_ids_with_protocol(protocol='public_UD_RGB', groups='dev')) == 25 # 10 model ids for dev
-#  assert len(db.model_ids_with_protocol(protocol='public_UD_RGB', groups='eval')) == 25 # 10 model ids for eval
-#
-#  assert len(db.objects(protocol='public_UD_RGB', groups='world')) == 1000 # 1000 samples in the world set
-#
-#  assert len(db.objects(protocol='public_UD_RGB', groups='dev', purposes='enroll')) == 500 # 500 samples for enrollment in the dev set (25 clients * 10 images * 2 recordings)
-#  assert len(db.objects(protocol='public_UD_RGB', groups='dev', purposes='enroll', model_ids='026')) == 20 # 20 samples to enroll model '026' in the dev set
-#  assert len(db.objects(protocol='public_UD_RGB', groups='dev', purposes='enroll', model_ids='071')) == 0 # 0 samples to enroll model '071' (it is an eval model)
-#  assert len(db.objects(protocol='public_UD_RGB', groups='dev', purposes='probe')) == 1000 # 1000 samples as probes in the dev set (25 clients * 10 images * 4 recordings)
-#
-#  assert len(db.objects(protocol='public_UD_RGB', groups='eval', purposes='enroll')) == 500 # 500 samples for enrollment in the eval set (25 clients * 10 images * 2 recordings)
-#  assert len(db.objects(protocol='public_UD_RGB', groups='eval', purposes='enroll', model_ids='058')) == 20 # 20 samples to enroll model '058' in the eval set
-#  assert len(db.objects(protocol='public_UD_RGB', groups='eval', purposes='enroll', model_ids='001')) == 0 # 0 samples to enroll model '001' (it is a world model)
-#  assert len(db.objects(protocol='public_UD_RGB', groups='eval', purposes='probe')) == 1000 # 500 samples as probes in the eval set (25 clients * 10 images * 4 recordings)
-#
-#def test_query_public_UO_RGB():
-#  """
-#  Test some queries for the public UO RGB protocol
-#  """
-#  db = Database()
-#  
-#  assert len(db.client_ids(protocol='public_UO_RGB')) == 75 # 75 client ids for world, dev and eval
-#  assert len(db.client_ids(protocol='public_UO_RGB', groups='world')) == 25 # 25 client ids for world
-#  assert len(db.client_ids(protocol='public_UO_RGB', groups='dev')) == 25 # 25 client ids for dev
-#  assert len(db.client_ids(protocol='public_UO_RGB', groups='eval')) == 25 # 25 client ids for eval
-#
-#  assert len(db.model_ids_with_protocol(protocol='public_UO_RGB')) == 75 # 75 model ids for world, dev and eval
-#  assert len(db.model_ids_with_protocol(protocol='public_UO_RGB', groups='world')) == 25 # 25 model ids for world
-#  assert len(db.model_ids_with_protocol(protocol='public_UO_RGB', groups='dev')) == 25 # 10 model ids for dev
-#  assert len(db.model_ids_with_protocol(protocol='public_UO_RGB', groups='eval')) == 25 # 10 model ids for eval
-#
-#  assert len(db.objects(protocol='public_UO_RGB', groups='world')) == 1000 # 1000 samples in the world set
-#
-#  assert len(db.objects(protocol='public_UO_RGB', groups='dev', purposes='enroll')) == 500 # 500 samples for enrollment in the dev set (25 clients * 10 images * 2 recordings)
-#  assert len(db.objects(protocol='public_UO_RGB', groups='dev', purposes='enroll', model_ids='026')) == 20 # 20 samples to enroll model '026' in the dev set
-#  assert len(db.objects(protocol='public_UO_RGB', groups='dev', purposes='enroll', model_ids='071')) == 0 # 0 samples to enroll model '071' (it is an eval model)
-#  assert len(db.objects(protocol='public_UO_RGB', groups='dev', purposes='probe')) == 1000 # 1000 samples as probes in the dev set (25 clients * 10 images * 4 recordings)
-#
-#  assert len(db.objects(protocol='public_UO_RGB', groups='eval', purposes='enroll')) == 500 # 500 samples for enrollment in the eval set (25 clients * 10 images * 2 recordings)
-#  assert len(db.objects(protocol='public_UO_RGB', groups='eval', purposes='enroll', model_ids='058')) == 20 # 20 samples to enroll model '058' in the eval set
-#  assert len(db.objects(protocol='public_UO_RGB', groups='eval', purposes='enroll', model_ids='001')) == 0 # 0 samples to enroll model '001' (it is a world model)
-#  assert len(db.objects(protocol='public_UO_RGB', groups='eval', purposes='probe')) == 1000 # 500 samples as probes in the eval set (25 clients * 10 images * 4 recordings)
-#
-#def test_query_public_NIR():
-#  """
-#  Test some queries for the public NIR protocols
-#  """
-#  db = Database()
-#  
-#  assert len(db.client_ids(protocol='public_MC_NIR')) == 75 # 75 client ids for world, dev and eval
-#  assert len(db.client_ids(protocol='public_MC_NIR', groups='dev')) == 25 # 25 client ids for dev
-#  assert len(db.model_ids_with_protocol(protocol='public_UD_NIR', groups='eval')) == 25 # 10 model ids for eval
-#  assert len(db.objects(protocol='public_UO_NIR', groups='world')) == 1000 # 1000 samples in the world set
-#  
-#  assert len(db.objects(protocol='public_UO_NIR', groups='dev', purposes='enroll')) == 500 # 500 samples for enrollment in the dev set (25 clients * 10 images * 2 recordings)
-#  assert len(db.objects(protocol='public_UD_NIR', groups='dev', purposes='enroll', model_ids='034')) == 20 # 20 samples to enroll model '034' in the dev set
-#  assert len(db.objects(protocol='public_UO_NIR', groups='dev', purposes='enroll', model_ids='062')) == 0 # 0 samples to enroll model '062' (it is an eval model)
-#  assert len(db.objects(protocol='public_MC_NIR', groups='dev', purposes='probe')) == 500 # MC Protocol -> 500 samples as probes in the dev set (25 clients * 10 images * 2 recordings)
-#  assert len(db.objects(protocol='public_UD_NIR', groups='dev', purposes='probe')) == 1000 # UO protocol -> 1000 samples as probes in the dev set (25 clients * 10 images * 4 recordings)
-#
-#  assert len(db.objects(protocol='public_UO_NIR', groups='eval', purposes='enroll')) == 500 # 500 samples for enrollment in the eval set (25 clients * 10 images * 2 recordings)
-#  assert len(db.objects(protocol='public_UD_NIR', groups='eval', purposes='enroll', model_ids='058')) == 20 # 20 samples to enroll model '058' in the eval set
-#  assert len(db.objects(protocol='public_UO_NIR', groups='eval', purposes='enroll', model_ids='001')) == 0 # 0 samples to enroll model '001' (it is a world model)
-#  assert len(db.objects(protocol='public_MC_NIR', groups='eval', purposes='probe')) == 500 # MC Protocol -> 500 samples as probes in the dev set (25 clients * 10 images * 4 recordings)
-#  assert len(db.objects(protocol='public_UD_NIR', groups='eval', purposes='probe')) == 1000 # 500 samples as probes in the eval set (25 clients * 10 images * 4 recordings)
-#  
-#def test_query_public_depth():
-#  """
-#  Test some queries for the public depth protocols
-#  """
-#  db = Database()
-#  
-#  assert len(db.client_ids(protocol='public_MC_depth')) == 75 # 75 client ids for world, dev and eval
-#  assert len(db.client_ids(protocol='public_MC_depth', groups='dev')) == 25 # 25 client ids for dev
-#  assert len(db.model_ids_with_protocol(protocol='public_UD_depth', groups='eval')) == 25 # 10 model ids for eval
-#  assert len(db.objects(protocol='public_UO_depth', groups='world')) == 1000 # 1000 samples in the world set
-#  
-#  assert len(db.objects(protocol='public_UO_depth', groups='dev', purposes='enroll')) == 500 # 500 samples for enrollment in the dev set (25 clients * 10 images * 2 recordings)
-#  assert len(db.objects(protocol='public_UD_depth', groups='dev', purposes='enroll', model_ids='034')) == 20 # 20 samples to enroll model '034' in the dev set
-#  assert len(db.objects(protocol='public_UO_depth', groups='dev', purposes='enroll', model_ids='062')) == 0 # 0 samples to enroll model '062' (it is an eval model)
-#  assert len(db.objects(protocol='public_MC_depth', groups='dev', purposes='probe')) == 500 # MC Protocol -> 500 samples as probes in the dev set (25 clients * 10 images * 4 recordings)
-#  assert len(db.objects(protocol='public_UD_depth', groups='dev', purposes='probe')) == 1000 # UO protocol -> 1000 samples as probes in the dev set (25 clients * 10 images * 4 recordings)
-#
-#  assert len(db.objects(protocol='public_UO_depth', groups='eval', purposes='enroll')) == 500 # 500 samples for enrollment in the eval set (25 clients * 10 images * 2 recordings)
-#  assert len(db.objects(protocol='public_UD_depth', groups='eval', purposes='enroll', model_ids='058')) == 20 # 20 samples to enroll model '058' in the eval set
-#  assert len(db.objects(protocol='public_UO_depth', groups='eval', purposes='enroll', model_ids='001')) == 0 # 0 samples to enroll model '001' (it is a world model)
-#  assert len(db.objects(protocol='public_MC_depth', groups='eval', purposes='probe')) == 500 # MC Protocol -> 500 samples as probes in the dev set (25 clients * 10 images * 4 recordings)
-#  #assert len(db.objects(protocol='public_UD_depth', groups='eval', purposes='probe')) == 1000 # 500 samples as probes in the eval set (25 clients * 10 images * 4 recordings)
+  assert len(db.objects(protocol='mc-rgb', groups='world')) == 1000
+  assert len(db.objects(protocol='mc-rgb', groups='dev', purposes='enroll')) == 500
+  assert len(db.objects(protocol='mc-rgb', groups='dev', purposes='enroll', model_ids=26)) == 20
+  assert len(db.objects(protocol='mc-rgb', groups='dev', purposes='probe')) == 500
+  assert len(db.objects(protocol='mc-rgb', groups='dev', purposes='probe', model_ids=26)) == 500 # dense probing
+  assert len(db.objects(protocol='mc-rgb', groups='eval', purposes='enroll')) == 500
+  assert len(db.objects(protocol='mc-rgb', groups='eval', purposes='enroll', model_ids=51)) == 20
+  assert len(db.objects(protocol='mc-rgb', groups='eval', purposes='probe')) == 500
+  assert len(db.objects(protocol='mc-rgb', groups='eval', purposes='probe', model_ids=51)) == 500 # dense probing
+
+  assert len(db.objects(protocol='ud-nir', groups='world')) == 1000
+  assert len(db.objects(protocol='ud-nir', groups='dev', purposes='enroll')) == 500
+  assert len(db.objects(protocol='ud-nir', groups='dev', purposes='enroll', model_ids=26)) == 20
+  assert len(db.objects(protocol='ud-nir', groups='dev', purposes='probe')) == 1000
+  assert len(db.objects(protocol='ud-nir', groups='dev', purposes='probe', model_ids=26)) == 1000 # dense probing
+  assert len(db.objects(protocol='ud-nir', groups='eval', purposes='enroll')) == 500
+  assert len(db.objects(protocol='ud-nir', groups='eval', purposes='enroll', model_ids=51)) == 20
+  assert len(db.objects(protocol='ud-nir', groups='eval', purposes='probe')) == 1000
+  assert len(db.objects(protocol='ud-nir', groups='eval', purposes='probe', model_ids=51)) == 1000 # dense probing
+
+  assert len(db.objects(protocol='uo-depth', groups='world')) == 1000
+  assert len(db.objects(protocol='uo-depth', groups='dev', purposes='enroll')) == 500
+  assert len(db.objects(protocol='uo-depth', groups='dev', purposes='enroll', model_ids=26)) == 20
+  assert len(db.objects(protocol='uo-depth', groups='dev', purposes='probe')) == 1000
+  assert len(db.objects(protocol='uo-depth', groups='dev', purposes='probe', model_ids=26)) == 1000 # dense probing
+  assert len(db.objects(protocol='uo-depth', groups='eval', purposes='enroll')) == 500
+  assert len(db.objects(protocol='uo-depth', groups='eval', purposes='enroll', model_ids=51)) == 20
+  assert len(db.objects(protocol='uo-depth', groups='eval', purposes='probe')) == 1000
+  assert len(db.objects(protocol='uo-depth', groups='eval', purposes='probe', model_ids=51)) == 1000 # dense probing
-- 
GitLab