test_databases.py 12.1 KB
Newer Older
Manuel Günther's avatar
Manuel Günther committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Manuel Guenther <Manuel.Guenther@idiap.ch>
# @date: Thu May 24 10:41:42 CEST 2012
#
# Copyright (C) 2011-2012 Idiap Research Institute, Martigny, Switzerland
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
21
import pytest
Manuel Günther's avatar
Manuel Günther committed
22

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
23
import os
Manuel Günther's avatar
Manuel Günther committed
24
import bob.bio.base
25
from bob.bio.base.test.utils import db_available
26
27
28
from bob.bio.base.test.test_database_implementations import (
    check_database,
)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
29
import bob.core
30
from bob.extension.download import get_file
31

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
32
logger = bob.core.log.setup("bob.bio.face")
Manuel Günther's avatar
Manuel Günther committed
33
34


35
36
37
def _check_annotations(
    database, topleft=False, required=True, limit_files=None, framed=False
):
38
39
    database_legacy = database.database
    files = database_legacy.all_files()
40
41
    if limit_files is not None:
        import random
42

43
        files = random.sample(files, limit_files)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
44
    found_none = False
45

46
    for file in files:
47
        annotations = database_legacy.annotations(file)
48
49
        if required:
            assert annotations is not None
50
51
        if annotations is not None:
            assert isinstance(annotations, dict)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
52
53
54
            if framed:
                # take one of the frames
                annotations = annotations[list(annotations.keys())[0]]
55
            if topleft:
56
57
                assert "topleft" in annotations
                assert "bottomright" in annotations
58
            else:
59
60
                assert "reye" in annotations
                assert "leye" in annotations
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
61
62
63
        else:
            found_none = True
    if found_none:
64
        logger.warn("Some annotations were None for {}".format(database_legacy.name))
Manuel Günther's avatar
Manuel Günther committed
65
66


67
@db_available("arface")
Manuel Günther's avatar
Manuel Günther committed
68
def test_arface():
69
    database = bob.bio.base.load_resource(
70
71
        "arface", "database", preferred_package="bob.bio.face"
    )
72
    try:
73
        check_database(database, groups=("dev", "eval"))
74
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
75
        pytest.skip("The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
76
            % e
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
77

78
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
79
80
81
    try:
        _check_annotations(database)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
82
        pytest.skip(
83
84
85
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
86
87


88
@db_available("atnt")
89
def test_atnt():
90
    database = bob.bio.base.load_resource(
91
92
        "atnt", "database", preferred_package="bob.bio.face"
    )
93
94
    try:
        check_database(database)
95
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
96
        pytest.skip(
97
98
99
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
Manuel Günther's avatar
Manuel Günther committed
100
101


102
@db_available("gbu")
Manuel Günther's avatar
Manuel Günther committed
103
def test_gbu():
104
    database = bob.bio.base.load_resource(
105
106
        "gbu", "database", preferred_package="bob.bio.face"
    )
107
    try:
108
        check_database(database, models_depend=True)
109
110
        check_database(database, protocol="Bad", models_depend=True)
        check_database(database, protocol="Ugly", models_depend=True)
111
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
112
        pytest.skip(
113
114
115
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
116
117
118
    try:
        _check_annotations(database, limit_files=1000)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
119
        pytest.skip(
120
121
122
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
123
124


125
@db_available("lfw")
Manuel Günther's avatar
Manuel Günther committed
126
def test_lfw():
127
    database = bob.bio.base.load_resource(
128
129
        "lfw-restricted", "database", preferred_package="bob.bio.face"
    )
130
    try:
131
        check_database(database, training_depends=True, models_depend=True)
132
133
134
135
136
137
138
139
140
141
142
143
144
145
        check_database(
            database,
            groups=("dev", "eval"),
            protocol="fold1",
            training_depends=True,
            models_depend=True,
        )
        check_database(
            bob.bio.base.load_resource(
                "lfw-unrestricted", "database", preferred_package="bob.bio.face"
            ),
            training_depends=True,
            models_depend=True,
        )
146
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
147
        pytest.skip(
148
149
150
            "The database could not queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
151
152
153
    try:
        _check_annotations(database, limit_files=1000)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
154
        pytest.skip(
155
156
157
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
Manuel Günther's avatar
Manuel Günther committed
158
159
160


def test_mobio():
161
    from bob.bio.face.database import MobioDatabase
162

163
164
165
166
167
168
169
    # Getting the absolute path
    urls = MobioDatabase.urls()
    filename = get_file("mobio.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
170
    except Exception:
171
172
        pass

173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
    protocols = MobioDatabase.protocols()
    for p in protocols:
        database = MobioDatabase(protocol=p)
        assert len(database.background_model_samples()) > 0
        assert len(database.treferences()) > 0
        assert len(database.zprobes()) > 0

        assert len(database.references(group="dev")) > 0
        assert len(database.probes(group="dev")) > 0

        assert len(database.references(group="eval")) > 0
        assert len(database.probes(group="eval")) > 0

    # Sanity check on mobio-male
    database = MobioDatabase(protocol="mobile0-male")
188
189
    assert len(database.treferences()) == 8
    assert len(database.zprobes()) == 960
190
191
192
193
194
195
196
    assert len(database.background_model_samples()) == 9600

    assert len(database.references()) == 24
    assert len(database.probes()) == 2520

    assert len(database.references(group="eval")) == 38
    assert len(database.probes(group="eval")) == 3990
Manuel Günther's avatar
Manuel Günther committed
197
198
199


def test_multipie():
200
    from bob.bio.face.database import MultipieDatabase
201

202
203
204
205
206
207
208
    # Getting the absolute path
    urls = MultipieDatabase.urls()
    filename = get_file("multipie.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
209
    except Exception:
210
211
        pass

212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
    protocols = MultipieDatabase.protocols()

    for p in protocols:
        database = MultipieDatabase(protocol=p)
        assert len(database.background_model_samples()) > 0

        assert len(database.references(group="dev")) > 0
        assert len(database.probes(group="dev")) > 0

        assert len(database.references(group="eval")) > 0
        assert len(database.probes(group="eval")) > 0

    database = MultipieDatabase(protocol="P")
    assert len(database.background_model_samples()) == 7725

    assert len(database.references(group="dev")) == 64
    assert len(database.probes(group="dev")) == 3328

    assert len(database.references(group="eval")) == 65
    assert len(database.probes(group="eval")) == 3380
Manuel Günther's avatar
Manuel Günther committed
232
233


234
@db_available("replay")
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
235
def test_replay_licit():
236
    database = bob.bio.base.load_resource(
237
238
        "replay-img-licit", "database", preferred_package="bob.bio.face"
    )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
239
    try:
240
        check_database(database, groups=("dev", "eval"))
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
241
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
242
        pytest.skip(
243
244
245
            "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
246
    try:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
247
248
        _check_annotations(database, topleft=True)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
249
        pytest.skip(
250
251
252
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
253
254


255
@db_available("replay")
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
256
def test_replay_spoof():
257
    database = bob.bio.base.load_resource(
258
259
        "replay-img-spoof", "database", preferred_package="bob.bio.face"
    )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
260
    try:
261
        check_database(database, groups=("dev", "eval"))
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
262
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
263
        pytest.skip(
264
265
266
            "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
267
    try:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
268
269
        _check_annotations(database, topleft=True)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
270
        pytest.skip(
271
272
273
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
274
275


276
@db_available("replaymobile")
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
277
def test_replaymobile_licit():
278
    database = bob.bio.base.load_resource(
279
280
        "replaymobile-img-licit", "database", preferred_package="bob.bio.face"
    )
281
    try:
282
        check_database(database, groups=("dev", "eval"))
283
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
284
        pytest.skip(
285
286
287
            "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
288
    try:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
289
290
        _check_annotations(database, topleft=True, limit_files=20)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
291
        pytest.skip(
292
293
294
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
295
296


297
@db_available("replaymobile")
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
298
def test_replaymobile_spoof():
299
    database = bob.bio.base.load_resource(
300
301
        "replaymobile-img-spoof", "database", preferred_package="bob.bio.face"
    )
302
    try:
303
        check_database(database, groups=("dev", "eval"))
304
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
305
        pytest.skip(
306
307
308
            "The database could not be queried; probably the db.sql3 file is missing. Here is the error: '%s'"
            % e
        )
309
    try:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
310
311
        _check_annotations(database, topleft=True, limit_files=20)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
312
        pytest.skip(
313
314
315
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
316
317


318
@db_available("ijbc")
319
320
def test_ijbc():
    database = bob.bio.base.load_resource(
321
322
        "ijbc-11", "database", preferred_package="bob.bio.face"
    )
323
324
325
    try:
        check_database(database, models_depend=True, training_depends=True)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
326
        pytest.skip("The database could not queried; Here is the error: '%s'" % e)
327
328
329
    try:
        _check_annotations(database, topleft=True, limit_files=1000)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
330
        pytest.skip(
331
332
333
            "The annotations could not be queried; probably the annotation files are missing. Here is the error: '%s'"
            % e
        )
334

335
336

@db_available("fargo")
337
338
def test_fargo():
    database = bob.bio.base.load_resource(
339
340
        "fargo", "database", preferred_package="bob.bio.face"
    )
341
342
343
    try:
        check_database(database)
    except IOError as e:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
344
        pytest.skip("The database could not queried; Here is the error: '%s'" % e)
345
346
347


def test_meds():
348

349
350
    from bob.bio.face.database import MEDSDatabase

351
352
353
354
355
356
357
    # Getting the absolute path
    urls = MEDSDatabase.urls()
    filename = get_file("meds.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
358
    except Exception:
359
360
        pass

361
362
363
    database = MEDSDatabase("verification_fold1")

    assert len(database.background_model_samples()) == 234
364
    assert len(database.references()) == 111
365
    assert len(database.probes()) == 313
366
367
368
369
370
371

    assert len(database.zprobes()) == 80
    assert len(database.treferences()) == 80

    assert len(database.references(group="eval")) == 112
    assert len(database.probes(group="eval")) == 309
372
373
374
375
376
377
378
379
380
381
382
383
384


def test_morph():

    from bob.bio.face.database import MorphDatabase

    # Getting the absolute path
    urls = MorphDatabase.urls()
    filename = get_file("morph.tar.gz", urls)

    # Removing the file before the test
    try:
        os.remove(filename)
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
385
    except Exception:
386
387
388
389
390
391
392
393
394
395
396
397
398
        pass

    database = MorphDatabase("verification_fold1")

    assert len(database.background_model_samples()) == 226
    assert len(database.references()) == 6738
    assert len(database.probes()) == 6557

    assert len(database.zprobes()) == 66
    assert len(database.treferences()) == 69

    assert len(database.references(group="eval")) == 6742
    assert len(database.probes(group="eval")) == 6553