test_databases.py 13.1 KB
Newer Older
Manuel Günther's avatar
Manuel Günther committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Thu May 24 10:41:42 CEST 2012
#
# Copyright (C) 2011-2012 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
21
import pytest
Manuel Günther's avatar
Manuel Günther committed
22

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
23
import os
Manuel Günther's avatar
Manuel Günther committed
24
import bob.bio.base
25
from bob.bio.base.test.utils import db_available
26
from bob.bio.base.test.test_database_implementations import check_database
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
27
import bob.core
28
from bob.extension.download import get_file
29
from nose.plugins.skip import SkipTest
30
from bob.extension import rc
31

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
32
logger = bob.core.log.setup("bob.bio.face")
Manuel Günther's avatar
Manuel Günther committed
33
34


35
36
37
def _check_annotations(
    database, topleft=False, required=True, limit_files=None, framed=False
):
38
39
    database_legacy = database.database
    files = database_legacy.all_files()
40
41
    if limit_files is not None:
        import random
42

43
        files = random.sample(files, limit_files)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
44
    found_none = False
45

46
    for file in files:
47
        annotations = database_legacy.annotations(file)
48
49
        if required:
            assert annotations is not None
50
51
        if annotations is not None:
            assert isinstance(annotations, dict)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
52
53
54
            if framed:
                # take one of the frames
                annotations = annotations[list(annotations.keys())[0]]
55
            if topleft:
56
57
                assert "topleft" in annotations
                assert "bottomright" in annotations
58
            else:
59
60
                assert "reye" in annotations
                assert "leye" in annotations
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
61
62
63
        else:
            found_none = True
    if found_none:
64
        logger.warn("Some annotations were None for {}".format(database_legacy.name))
Manuel Günther's avatar
Manuel Günther committed
65
66


67
@db_available("arface")
Manuel Günther's avatar
Manuel Günther committed
68
def test_arface():
69
    database = bob.bio.base.load_resource(
70
71
        "arface", "database", preferred_package="bob.bio.face"
    )
72
    try:
73
        check_database(database, groups=("dev", "eval"))
74
    except IOError as e:
75
76
        pytest.skip(
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
77
78
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
79
80
81
    try:
        _check_annotations(database)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
82
        pytest.skip(
83
84
85
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
86
87


88
@db_available("atnt")
89
def test_atnt():
90
    database = bob.bio.base.load_resource(
91
92
        "atnt", "database", preferred_package="bob.bio.face"
    )
93
94
    try:
        check_database(database)
95
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
96
        pytest.skip(
97
98
99
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
Manuel Günther's avatar
Manuel Günther committed
100
101


102
@db_available("gbu")
Manuel Günther's avatar
Manuel Günther committed
103
def test_gbu():
104
    database = bob.bio.base.load_resource(
105
106
        "gbu", "database", preferred_package="bob.bio.face"
    )
107
    try:
108
        check_database(database, models_depend=True)
109
110
        check_database(database, protocol="Bad", models_depend=True)
        check_database(database, protocol="Ugly", models_depend=True)
111
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
112
        pytest.skip(
113
114
115
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
116
117
118
    try:
        _check_annotations(database, limit_files=1000)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
119
        pytest.skip(
120
121
122
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
123
124


125
@db_available("lfw")
Manuel Günther's avatar
Manuel Günther committed
126
def test_lfw():
127
    database = bob.bio.base.load_resource(
128
129
        "lfw-restricted", "database", preferred_package="bob.bio.face"
    )
130
    try:
131
        check_database(database, training_depends=True, models_depend=True)
132
133
134
135
136
137
138
139
140
141
142
143
144
145
        check_database(
            database,
            groups=("dev", "eval"),
            protocol="fold1",
            training_depends=True,
            models_depend=True,
        )
        check_database(
            bob.bio.base.load_resource(
                "lfw-unrestricted", "database", preferred_package="bob.bio.face"
            ),
            training_depends=True,
            models_depend=True,
        )
146
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
147
        pytest.skip(
148
149
150
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
151
152
153
    try:
        _check_annotations(database, limit_files=1000)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
154
        pytest.skip(
155
156
157
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
Manuel Günther's avatar
Manuel Günther committed
158
159
160


def test_mobio():
161
    from bob.bio.face.database import MobioDatabase
162

163
164
165
166
167
168
169
    # Getting the absolute path
    urls = MobioDatabase.urls()
    filename = get_file("mobio.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
170
    except Exception:
171
172
        pass

173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
    protocols = MobioDatabase.protocols()
    for p in protocols:
        database = MobioDatabase(protocol=p)
        assert len(database.background_model_samples()) > 0
        assert len(database.treferences()) > 0
        assert len(database.zprobes()) > 0

        assert len(database.references(group="dev")) > 0
        assert len(database.probes(group="dev")) > 0

        assert len(database.references(group="eval")) > 0
        assert len(database.probes(group="eval")) > 0

    # Sanity check on mobio-male
    database = MobioDatabase(protocol="mobile0-male")
188
189
    assert len(database.treferences()) == 8
    assert len(database.zprobes()) == 960
190
191
192
193
194
195
196
    assert len(database.background_model_samples()) == 9600

    assert len(database.references()) == 24
    assert len(database.probes()) == 2520

    assert len(database.references(group="eval")) == 38
    assert len(database.probes(group="eval")) == 3990
Manuel Günther's avatar
Manuel Günther committed
197
198
199


def test_multipie():
200
    from bob.bio.face.database import MultipieDatabase
201

202
203
204
205
206
207
208
    # Getting the absolute path
    urls = MultipieDatabase.urls()
    filename = get_file("multipie.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
209
    except Exception:
210
211
        pass

212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
    protocols = MultipieDatabase.protocols()

    for p in protocols:
        database = MultipieDatabase(protocol=p)
        assert len(database.background_model_samples()) > 0

        assert len(database.references(group="dev")) > 0
        assert len(database.probes(group="dev")) > 0

        assert len(database.references(group="eval")) > 0
        assert len(database.probes(group="eval")) > 0

    database = MultipieDatabase(protocol="P")
    assert len(database.background_model_samples()) == 7725

    assert len(database.references(group="dev")) == 64
    assert len(database.probes(group="dev")) == 3328

    assert len(database.references(group="eval")) == 65
    assert len(database.probes(group="eval")) == 3380
Manuel Günther's avatar
Manuel Günther committed
232
233


234
@db_available("replay")
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
235
def test_replay_licit():
236
    database = bob.bio.base.load_resource(
237
238
        "replay-img-licit", "database", preferred_package="bob.bio.face"
    )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
239
    try:
240
        check_database(database, groups=("dev", "eval"))
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
241
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
242
        pytest.skip(
243
244
245
            "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
246
    try:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
247
248
        _check_annotations(database, topleft=True)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
249
        pytest.skip(
250
251
252
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
253
254


255
@db_available("replay")
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
256
def test_replay_spoof():
257
    database = bob.bio.base.load_resource(
258
259
        "replay-img-spoof", "database", preferred_package="bob.bio.face"
    )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
260
    try:
261
        check_database(database, groups=("dev", "eval"))
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
262
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
263
        pytest.skip(
264
265
266
            "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
267
    try:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
268
269
        _check_annotations(database, topleft=True)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
270
        pytest.skip(
271
272
273
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
274
275


276
def test_replaymobile():
277
    database = bob.bio.base.load_resource(
278
        "replaymobile-img", "database", preferred_package="bob.bio.face"
279
    )
280
281
282
283
284
285
286
287
288
289
290
291
    samples = database.all_samples(groups=("dev", "eval"))
    assert len(samples) == 8300, len(samples)
    sample = samples[0]
    assert hasattr(sample, "annotations")
    assert "reye" in sample.annotations
    assert "leye" in sample.annotations
    assert hasattr(sample, "path")
    assert hasattr(sample, "frame")
    assert len(database.references()) == 16
    assert len(database.references(group="eval")) == 12
    assert len(database.probes()) == 4160
    assert len(database.probes(group="eval")) == 3020
292
    try:
293
294
295
296
297
298
299
300
301
302
303
304
305
        assert sample.annotations == {
            "bottomright": [785, 395],
            "topleft": [475, 167],
            "leye": [587, 336],
            "reye": [588, 238],
            "mouthleft": [705, 252],
            "mouthright": [706, 326],
            "nose": [643, 295],
        }
        assert sample.data.shape == (3, 1280, 720)
        assert sample.data[0, 0, 0] == 87
    except RuntimeError as e:
        raise SkipTest(e)
306
307


308
309
310
311
@pytest.mark.skipif(
    rc.get("bob.bio.face.ijbc.directory") is None,
    reason="IJBC original protocols not available. Please do `bob config set bob.bio.face.ijbc.directory [IJBC PATH]` to set the IJBC data path.",
)
312
def test_ijbc():
313
314
315
316
    from bob.bio.face.database import IJBCDatabase

    database = IJBCDatabase()

317
    # assert len(database.background_model_samples()) == 140732
318
319
    assert len(database.references()) == 3531
    assert len(database.probes()) == 19593
320
321
    num_comparisons = sum([len(item.references) for item in database.probes()])
    assert num_comparisons == 19557 + 15638932 # Genuine + Impostor
322

323
324

@db_available("fargo")
325
326
def test_fargo():
    database = bob.bio.base.load_resource(
327
328
        "fargo", "database", preferred_package="bob.bio.face"
    )
329
330
331
    try:
        check_database(database)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
332
        pytest.skip("The database could not queried; Here is the error: '%s'" % e)
333
334
335


def test_meds():
336

337
338
    from bob.bio.face.database import MEDSDatabase

339
340
341
342
343
344
345
    # Getting the absolute path
    urls = MEDSDatabase.urls()
    filename = get_file("meds.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
346
    except Exception:
347
348
        pass

349
350
351
    database = MEDSDatabase("verification_fold1")

    assert len(database.background_model_samples()) == 234
352
    assert len(database.references()) == 111
353
    assert len(database.probes()) == 313
354
355
356
357
358
359

    assert len(database.zprobes()) == 80
    assert len(database.treferences()) == 80

    assert len(database.references(group="eval")) == 112
    assert len(database.probes(group="eval")) == 309
360
361
362
363
364
365
366
367
368
369
370
371
372


def test_morph():

    from bob.bio.face.database import MorphDatabase

    # Getting the absolute path
    urls = MorphDatabase.urls()
    filename = get_file("morph.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
373
    except Exception:
374
375
376
377
378
379
380
381
382
383
384
385
386
        pass

    database = MorphDatabase("verification_fold1")

    assert len(database.background_model_samples()) == 226
    assert len(database.references()) == 6738
    assert len(database.probes()) == 6557

    assert len(database.zprobes()) == 66
    assert len(database.treferences()) == 69

    assert len(database.references(group="eval")) == 6742
    assert len(database.probes(group="eval")) == 6553
387
388
389
390
391
392
393
394
395
396


def test_casia_africa():

    from bob.bio.face.database import CasiaAfricaDatabase

    database = CasiaAfricaDatabase("ID-V-All-Ep1")

    assert len(database.references()) == 2455
    assert len(database.probes()) == 2426
397
398


Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
def test_frgc():

    from bob.bio.face.database import FRGCDatabase

    database = FRGCDatabase("2.0.1")

    assert len(database.background_model_samples()) == 12776
    assert len(database.references()) == 7572
    assert len(database.probes()) == 8456

    database = FRGCDatabase("2.0.2")
    assert len(database.background_model_samples()) == 12776
    assert len(database.references()) == 1893
    assert len(database.probes()) == 8456

    database = FRGCDatabase("2.0.4")
    assert len(database.background_model_samples()) == 12776
    assert len(database.references()) == 7572
    assert len(database.probes()) == 4228


420
def test_polathermal():
421
422
423
424
425
426

    from bob.bio.face.database import PolaThermalDatabase

    database = PolaThermalDatabase("VIS-thermal-overall-split1")
    assert len(database.references()) == 35
    assert len(database.probes()) == 1680
427
428
429
430
431
432
433
434
435
436


def test_cbsr_nir_vis_2():

    from bob.bio.face.database import CBSRNirVis2Database

    database = CBSRNirVis2Database("view2_1")

    assert len(database.references()) == 358
    assert len(database.probes()) == 6123