test_databases.py 13 KB
Newer Older
Manuel Günther's avatar
Manuel Günther committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Thu May 24 10:41:42 CEST 2012
#
# Copyright (C) 2011-2012 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
21
import pytest
Manuel Günther's avatar
Manuel Günther committed
22

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
23
import os
Manuel Günther's avatar
Manuel Günther committed
24
import bob.bio.base
25
from bob.bio.base.test.utils import db_available
26
from bob.bio.base.test.test_database_implementations import check_database
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
27
import bob.core
28
from bob.extension.download import get_file
29
from nose.plugins.skip import SkipTest
30

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
31
logger = bob.core.log.setup("bob.bio.face")
Manuel Günther's avatar
Manuel Günther committed
32
33


34
35
36
def _check_annotations(
    database, topleft=False, required=True, limit_files=None, framed=False
):
37
38
    database_legacy = database.database
    files = database_legacy.all_files()
39
40
    if limit_files is not None:
        import random
41

42
        files = random.sample(files, limit_files)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
43
    found_none = False
44

45
    for file in files:
46
        annotations = database_legacy.annotations(file)
47
48
        if required:
            assert annotations is not None
49
50
        if annotations is not None:
            assert isinstance(annotations, dict)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
51
52
53
            if framed:
                # take one of the frames
                annotations = annotations[list(annotations.keys())[0]]
54
            if topleft:
55
56
                assert "topleft" in annotations
                assert "bottomright" in annotations
57
            else:
58
59
                assert "reye" in annotations
                assert "leye" in annotations
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
60
61
62
        else:
            found_none = True
    if found_none:
63
        logger.warn("Some annotations were None for {}".format(database_legacy.name))
Manuel Günther's avatar
Manuel Günther committed
64
65


66
@db_available("arface")
Manuel Günther's avatar
Manuel Günther committed
67
def test_arface():
68
    database = bob.bio.base.load_resource(
69
70
        "arface", "database", preferred_package="bob.bio.face"
    )
71
    try:
72
        check_database(database, groups=("dev", "eval"))
73
    except IOError as e:
74
75
        pytest.skip(
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
76
77
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
78
79
80
    try:
        _check_annotations(database)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
81
        pytest.skip(
82
83
84
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
85
86


87
@db_available("atnt")
88
def test_atnt():
89
    database = bob.bio.base.load_resource(
90
91
        "atnt", "database", preferred_package="bob.bio.face"
    )
92
93
    try:
        check_database(database)
94
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
95
        pytest.skip(
96
97
98
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
Manuel Günther's avatar
Manuel Günther committed
99
100


101
@db_available("gbu")
Manuel Günther's avatar
Manuel Günther committed
102
def test_gbu():
103
    database = bob.bio.base.load_resource(
104
105
        "gbu", "database", preferred_package="bob.bio.face"
    )
106
    try:
107
        check_database(database, models_depend=True)
108
109
        check_database(database, protocol="Bad", models_depend=True)
        check_database(database, protocol="Ugly", models_depend=True)
110
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
111
        pytest.skip(
112
113
114
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
115
116
117
    try:
        _check_annotations(database, limit_files=1000)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
118
        pytest.skip(
119
120
121
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
122
123


124
@db_available("lfw")
Manuel Günther's avatar
Manuel Günther committed
125
def test_lfw():
126
    database = bob.bio.base.load_resource(
127
128
        "lfw-restricted", "database", preferred_package="bob.bio.face"
    )
129
    try:
130
        check_database(database, training_depends=True, models_depend=True)
131
132
133
134
135
136
137
138
139
140
141
142
143
144
        check_database(
            database,
            groups=("dev", "eval"),
            protocol="fold1",
            training_depends=True,
            models_depend=True,
        )
        check_database(
            bob.bio.base.load_resource(
                "lfw-unrestricted", "database", preferred_package="bob.bio.face"
            ),
            training_depends=True,
            models_depend=True,
        )
145
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
146
        pytest.skip(
147
148
149
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
150
151
152
    try:
        _check_annotations(database, limit_files=1000)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
153
        pytest.skip(
154
155
156
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
Manuel Günther's avatar
Manuel Günther committed
157
158
159


def test_mobio():
160
    from bob.bio.face.database import MobioDatabase
161

162
163
164
165
166
167
168
    # Getting the absolute path
    urls = MobioDatabase.urls()
    filename = get_file("mobio.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
169
    except Exception:
170
171
        pass

172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
    protocols = MobioDatabase.protocols()
    for p in protocols:
        database = MobioDatabase(protocol=p)
        assert len(database.background_model_samples()) > 0
        assert len(database.treferences()) > 0
        assert len(database.zprobes()) > 0

        assert len(database.references(group="dev")) > 0
        assert len(database.probes(group="dev")) > 0

        assert len(database.references(group="eval")) > 0
        assert len(database.probes(group="eval")) > 0

    # Sanity check on mobio-male
    database = MobioDatabase(protocol="mobile0-male")
187
188
    assert len(database.treferences()) == 8
    assert len(database.zprobes()) == 960
189
190
191
192
193
194
195
    assert len(database.background_model_samples()) == 9600

    assert len(database.references()) == 24
    assert len(database.probes()) == 2520

    assert len(database.references(group="eval")) == 38
    assert len(database.probes(group="eval")) == 3990
Manuel Günther's avatar
Manuel Günther committed
196
197
198


def test_multipie():
199
    from bob.bio.face.database import MultipieDatabase
200

201
202
203
204
205
206
207
    # Getting the absolute path
    urls = MultipieDatabase.urls()
    filename = get_file("multipie.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
208
    except Exception:
209
210
        pass

211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
    protocols = MultipieDatabase.protocols()

    for p in protocols:
        database = MultipieDatabase(protocol=p)
        assert len(database.background_model_samples()) > 0

        assert len(database.references(group="dev")) > 0
        assert len(database.probes(group="dev")) > 0

        assert len(database.references(group="eval")) > 0
        assert len(database.probes(group="eval")) > 0

    database = MultipieDatabase(protocol="P")
    assert len(database.background_model_samples()) == 7725

    assert len(database.references(group="dev")) == 64
    assert len(database.probes(group="dev")) == 3328

    assert len(database.references(group="eval")) == 65
    assert len(database.probes(group="eval")) == 3380
Manuel Günther's avatar
Manuel Günther committed
231
232


233
@db_available("replay")
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
234
def test_replay_licit():
235
    database = bob.bio.base.load_resource(
236
237
        "replay-img-licit", "database", preferred_package="bob.bio.face"
    )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
238
    try:
239
        check_database(database, groups=("dev", "eval"))
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
240
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
241
        pytest.skip(
242
243
244
            "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
245
    try:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
246
247
        _check_annotations(database, topleft=True)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
248
        pytest.skip(
249
250
251
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
252
253


254
@db_available("replay")
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
255
def test_replay_spoof():
256
    database = bob.bio.base.load_resource(
257
258
        "replay-img-spoof", "database", preferred_package="bob.bio.face"
    )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
259
    try:
260
        check_database(database, groups=("dev", "eval"))
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
261
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
262
        pytest.skip(
263
264
265
            "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
266
    try:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
267
268
        _check_annotations(database, topleft=True)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
269
        pytest.skip(
270
271
272
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
273
274


275
def test_replaymobile():
276
    database = bob.bio.base.load_resource(
277
        "replaymobile-img", "database", preferred_package="bob.bio.face"
278
    )
279
280
281
282
283
284
285
286
287
288
289
290
    samples = database.all_samples(groups=("dev", "eval"))
    assert len(samples) == 8300, len(samples)
    sample = samples[0]
    assert hasattr(sample, "annotations")
    assert "reye" in sample.annotations
    assert "leye" in sample.annotations
    assert hasattr(sample, "path")
    assert hasattr(sample, "frame")
    assert len(database.references()) == 16
    assert len(database.references(group="eval")) == 12
    assert len(database.probes()) == 4160
    assert len(database.probes(group="eval")) == 3020
291
    try:
292
293
294
295
296
297
298
299
300
301
302
303
304
        assert sample.annotations == {
            "bottomright": [785, 395],
            "topleft": [475, 167],
            "leye": [587, 336],
            "reye": [588, 238],
            "mouthleft": [705, 252],
            "mouthright": [706, 326],
            "nose": [643, 295],
        }
        assert sample.data.shape == (3, 1280, 720)
        assert sample.data[0, 0, 0] == 87
    except RuntimeError as e:
        raise SkipTest(e)
305
306


307
def test_ijbc():
308
309
310
311
312
313
314
    from bob.bio.face.database import IJBCDatabase

    # Getting the absolute path
    urls = IJBCDatabase.urls()
    filename = get_file("ijbc.tar.gz", urls)

    # Removing the file before the test
315
    try:
316
317
318
319
320
321
322
323
324
        os.remove(filename)
    except Exception:
        pass

    database = IJBCDatabase()

    assert len(database.background_model_samples()) == 140732
    assert len(database.references()) == 3531
    assert len(database.probes()) == 19593
325

326
327

@db_available("fargo")
328
329
def test_fargo():
    database = bob.bio.base.load_resource(
330
331
        "fargo", "database", preferred_package="bob.bio.face"
    )
332
333
334
    try:
        check_database(database)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
335
        pytest.skip("The database could not queried; Here is the error: '%s'" % e)
336
337
338


def test_meds():
339

340
341
    from bob.bio.face.database import MEDSDatabase

342
343
344
345
346
347
348
    # Getting the absolute path
    urls = MEDSDatabase.urls()
    filename = get_file("meds.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
349
    except Exception:
350
351
        pass

352
353
354
    database = MEDSDatabase("verification_fold1")

    assert len(database.background_model_samples()) == 234
355
    assert len(database.references()) == 111
356
    assert len(database.probes()) == 313
357
358
359
360
361
362

    assert len(database.zprobes()) == 80
    assert len(database.treferences()) == 80

    assert len(database.references(group="eval")) == 112
    assert len(database.probes(group="eval")) == 309
363
364
365
366
367
368
369
370
371
372
373
374
375


def test_morph():

    from bob.bio.face.database import MorphDatabase

    # Getting the absolute path
    urls = MorphDatabase.urls()
    filename = get_file("morph.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
376
    except Exception:
377
378
379
380
381
382
383
384
385
386
387
388
389
        pass

    database = MorphDatabase("verification_fold1")

    assert len(database.background_model_samples()) == 226
    assert len(database.references()) == 6738
    assert len(database.probes()) == 6557

    assert len(database.zprobes()) == 66
    assert len(database.treferences()) == 69

    assert len(database.references(group="eval")) == 6742
    assert len(database.probes(group="eval")) == 6553
390
391
392
393
394
395
396
397
398
399


def test_casia_africa():

    from bob.bio.face.database import CasiaAfricaDatabase

    database = CasiaAfricaDatabase("ID-V-All-Ep1")

    assert len(database.references()) == 2455
    assert len(database.probes()) == 2426
400
401


Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
def test_frgc():

    from bob.bio.face.database import FRGCDatabase

    database = FRGCDatabase("2.0.1")

    assert len(database.background_model_samples()) == 12776
    assert len(database.references()) == 7572
    assert len(database.probes()) == 8456

    database = FRGCDatabase("2.0.2")
    assert len(database.background_model_samples()) == 12776
    assert len(database.references()) == 1893
    assert len(database.probes()) == 8456

    database = FRGCDatabase("2.0.4")
    assert len(database.background_model_samples()) == 12776
    assert len(database.references()) == 7572
    assert len(database.probes()) == 4228


423
def test_polathermal():
424
425
426
427
428
429

    from bob.bio.face.database import PolaThermalDatabase

    database = PolaThermalDatabase("VIS-thermal-overall-split1")
    assert len(database.references()) == 35
    assert len(database.probes()) == 1680
430
431
432
433
434
435
436
437
438
439


def test_cbsr_nir_vis_2():

    from bob.bio.face.database import CBSRNirVis2Database

    database = CBSRNirVis2Database("view2_1")

    assert len(database.references()) == 358
    assert len(database.probes()) == 6123