test_baselines.py 4.08 KB
Newer Older
1
2
import pkg_resources
import numpy as np
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
3
from bob.pipelines import Sample, SampleSet, DelayedSample
4
from bob.bio.base import load_resource
5
6
7
8
from bob.bio.base.pipelines.vanilla_biometrics import (
    checkpoint_vanilla_biometrics,
    dask_vanilla_biometrics,
)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
9
10
11
12
13
import tempfile
import os
import bob.io.base
import functools
import copy
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
14
import pytest
15

16
from bob.bio.base.test.utils import is_library_available
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
17

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
images = dict()
images["bioref"] = (
    pkg_resources.resource_filename("bob.bio.face.test", "data/testimage.jpg"),
    {"reye": (131, 176), "leye": (222, 170)},
)
images["probe"] = (
    pkg_resources.resource_filename("bob.bio.face.test", "data/ada.png"),
    {"reye": (440, 207), "leye": (546, 207)},
)


def get_fake_sample_set(face_size=(160, 160), purpose="bioref"):

    data = images[purpose][0]
    annotations = images[purpose][1]
    key = "1" if purpose == "bioref" else "2"
34

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
35
36
    return [
        SampleSet(
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
37
38
39
40
41
42
43
44
            [
                DelayedSample(
                    load=functools.partial(bob.io.base.load, data),
                    key=key,
                    annotations=annotations,
                )
            ],
            key=key,
45
            reference_id=key,
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
46
47
48
            references=["1"],
        )
    ]
49
50


51
52
53
54
def get_fake_samples_for_training():

    data = np.random.rand(10, 3, 400, 400)
    annotations = {"reye": (131, 176), "leye": (222, 170)}
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
55

56
    return [
57
        Sample(x, key=str(i), reference_id=str(i), annotations=annotations)
58
        for i, x in enumerate(data)
59
60
61
    ]


62
def run_baseline(baseline, samples_for_training=[]):
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
63
64
    biometric_references = get_fake_sample_set(purpose="bioref")
    probes = get_fake_sample_set(purpose="probe")
65

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
66
    # Regular pipeline
67
    pipeline = load_resource(baseline, "pipeline")
68
    scores = pipeline(samples_for_training, biometric_references, probes)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
69
70
    assert len(scores) == 1
    assert len(scores[0]) == 1
71

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
72
73
    # CHECKPOINTING
    with tempfile.TemporaryDirectory() as d:
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
74

75
        cpy = copy.deepcopy(pipeline)
76
        checkpoint_pipeline = checkpoint_vanilla_biometrics(cpy, base_dir=d)
77

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
78
79
80
81
        checkpoint_scores = checkpoint_pipeline([], biometric_references, probes)
        assert len(checkpoint_scores) == 1
        assert len(checkpoint_scores[0]) == 1
        assert np.isclose(scores[0][0].data, checkpoint_scores[0][0].data)
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
82

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
83
84
85
86
87
        dirs = os.listdir(d)
        assert "biometric_references" in dirs
        assert "samplewrapper-1" in dirs
        assert "samplewrapper-2" in dirs
        assert "scores" in dirs
88

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
89
    # DASK
90

91
92
    with tempfile.TemporaryDirectory() as d:
        cpy = copy.deepcopy(pipeline)
93
        dask_pipeline = dask_vanilla_biometrics(
94
            checkpoint_vanilla_biometrics(cpy, base_dir=d)
95
        )
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
96
        dask_scores = dask_pipeline([], biometric_references, probes)
97
        dask_scores = dask_scores.compute(scheduler="single-threaded")
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
98
99
100
        assert len(dask_scores) == 1
        assert len(dask_scores[0]) == 1
        assert np.isclose(scores[0][0].data, dask_scores[0][0].data)
101

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
102
103
104
105
106
        dirs = os.listdir(d)
        assert "biometric_references" in dirs
        assert "samplewrapper-1" in dirs
        assert "samplewrapper-2" in dirs
        assert "scores" in dirs
107

108

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
109
@pytest.mark.slow
110
@is_library_available("tensorflow")
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
111
def test_facenet_baseline():
112
    run_baseline("facenet-sanderberg")
113

114

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
115
@pytest.mark.slow
116
@is_library_available("tensorflow")
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
117
def test_inception_resnetv2_msceleb():
118
    run_baseline("inception-resnetv2-msceleb")
119

120

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
121
@pytest.mark.slow
122
@is_library_available("tensorflow")
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
123
def test_inception_resnetv2_casiawebface():
124
    run_baseline("inception-resnetv2-casiawebface")
125
126


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
127
@pytest.mark.slow
128
@is_library_available("tensorflow")
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
129
def test_inception_resnetv1_msceleb():
130
    run_baseline("inception-resnetv1-msceleb")
131
132


Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
133
@pytest.mark.slow
134
@is_library_available("tensorflow")
135
def test_inception_resnetv1_casiawebface():
136
    run_baseline("inception-resnetv1-casiawebface")
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
137

138

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
139
@pytest.mark.slow
140
@is_library_available("mxnet")
141
142
def test_arcface_insightface():
    run_baseline("arcface-insightface")
Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
143

144

Tiago de Freitas Pereira's avatar
Tiago de Freitas Pereira committed
145
def test_gabor_graph():
146
147
148
    run_baseline("gabor_graph")


149
# def test_lda():
150
#    run_baseline("lda", get_fake_samples_for_training())