test_script.py 8.48 KB
Newer Older
1 2
'''Tests for bob.measure scripts'''

3
import sys
4 5 6 7
import filecmp
import click
from click.testing import CliRunner
import bob.io.base.test_utils
8
from .script import commands
9
from bob.extension.scripts.click_helper import assert_click_runner_result
10 11 12 13

def test_metrics():
    dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
    runner = CliRunner()
14
    result = runner.invoke(commands.metrics, [dev1])
15 16 17 18
    with runner.isolated_filesystem():
        with open('tmp', 'w') as f:
            f.write(result.output)
        test_ref = bob.io.base.test_utils.datafile('test_m1.txt', 'bob.measure')
19
        assert_click_runner_result(result)
20 21 22 23 24 25

    dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
    test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
    test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
    with runner.isolated_filesystem():
        result = runner.invoke(
26
            commands.metrics, ['-e', dev1, test1, dev2, test2]
27 28 29 30
        )
        with open('tmp', 'w') as f:
            f.write(result.output)
        test_ref = bob.io.base.test_utils.datafile('test_m2.txt', 'bob.measure')
31
        assert_click_runner_result(result)
32 33
    with runner.isolated_filesystem():
        result = runner.invoke(
34
            commands.metrics, ['-e', '-l', 'tmp', dev1, test1, dev2, test2, '-lg',
35
                              'A,B']
36
        )
37
        assert_click_runner_result(result)
38

39 40
    with runner.isolated_filesystem():
        result = runner.invoke(
41
            commands.metrics, ['-l', 'tmp', dev1, dev2]
42
        )
43
        assert_click_runner_result(result)
44 45 46 47 48

def test_roc():
    dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
    runner = CliRunner()
    with runner.isolated_filesystem():
49
        result = runner.invoke(commands.roc, ['--output', 'test.pdf',dev1])
50 51
        if result.output:
            click.echo(result.output)
52
        assert_click_runner_result(result)
53 54 55 56
    dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
    test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
    test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
    with runner.isolated_filesystem():
57
        result = runner.invoke(commands.roc, ['--split', '--output',
58
                                              'test.pdf', '-e',
59
                                              '-ts', 'A,',
60 61 62
                                              dev1, test1, dev2, test2])
        if result.output:
            click.echo(result.output)
63
        assert_click_runner_result(result)
64

65
    with runner.isolated_filesystem():
66
        result = runner.invoke(commands.roc, ['-e', '--output',
67
                                              'test.pdf', '--legends', 'A,B',
68 69 70
                                              dev1, test1, dev2, test2])
        if result.output:
            click.echo(result.output)
71
        assert_click_runner_result(result)
72 73


74 75 76 77
def test_det():
    dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
    runner = CliRunner()
    with runner.isolated_filesystem():
78
        result = runner.invoke(commands.det, [dev1])
79 80
        if result.output:
            click.echo(result.output)
81
        assert_click_runner_result(result)
82 83 84
    dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
    test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
    test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
85
    with runner.isolated_filesystem():
86
        result = runner.invoke(commands.det, ['-e', '--split', '--output',
87
                                              'test.pdf', '--legends', 'A,B',
88
                                              '-ll', 'upper-right',
89 90 91
                                              dev1, test1, dev2, test2])
        if result.output:
            click.echo(result.output)
92
        assert_click_runner_result(result)
93

94
    with runner.isolated_filesystem():
95
        result = runner.invoke(commands.det, ['--output',
96
                                              'test.pdf', '-e',
97 98 99
                                              dev1, test1, dev2, test2])
        if result.output:
            click.echo(result.output)
100
        assert_click_runner_result(result)
101 102 103 104 105 106

def test_epc():
    dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
    test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
    runner = CliRunner()
    with runner.isolated_filesystem():
107
        result = runner.invoke(commands.epc, [dev1, test1])
108 109
        if result.output:
            click.echo(result.output)
110
        assert_click_runner_result(result)
111

112 113 114
    dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
    test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
    with runner.isolated_filesystem():
115
        result = runner.invoke(commands.epc, ['--output', 'test.pdf',
116
                                              '--legends', 'A,B',
117
                                              '--titles', 'TA,TB',
118
                                              '-ll', 'upper-right',
119 120 121
                                              dev1, test1, dev2, test2])
        if result.output:
            click.echo(result.output)
122
        assert_click_runner_result(result)
123 124 125 126 127 128 129 130

def test_hist():
    dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
    test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
    dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
    test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
    runner = CliRunner()
    with runner.isolated_filesystem():
131
        result = runner.invoke(commands.hist, [dev1])
132 133
        if result.output:
            click.echo(result.output)
134
        assert_click_runner_result(result)
135 136

    with runner.isolated_filesystem():
137 138
        result = runner.invoke(commands.hist, ['--criterion',
                                               'min-hter', '--no-line',
139
                                               '--output', 'HISTO.pdf',  '-b',
140
                                               '30,100', dev1, dev2])
141 142
        if result.output:
            click.echo(result.output)
143
        assert_click_runner_result(result)
144 145

    with runner.isolated_filesystem():
146
        result = runner.invoke(commands.hist, ['-e', '--criterion', 'eer','--output',
147
                                               'HISTO.pdf',  '-b',  '30,20',
148
                                               '-sp', 111, '-ts', 'A,B',
149 150 151
                                               dev1, test1, dev2, test2])
        if result.output:
            click.echo(result.output)
152
        assert_click_runner_result(result)
153

154 155 156 157 158 159 160
def test_hist_legends():
    dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
    test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
    dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
    test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
    runner = CliRunner()

161
    # share same title for dev/eval of each system
162
    with runner.isolated_filesystem():
163
        result = runner.invoke(commands.hist, ['-e', '-sp', 111, '-ts', 'A,B',
164 165 166
                                               dev1, test1, dev2, test2])
        if result.output:
            click.echo(result.output)
167
        assert_click_runner_result(result)
168

169
    # individual titles for dev and eval
170
    with runner.isolated_filesystem():
171
        result = runner.invoke(commands.hist, ['-e', '-sp', 121, '-ts',
172 173 174 175
                                               'A,B,C,D',
                                               dev1, test1, dev2, test2])
        if result.output:
            click.echo(result.output)
176
        assert_click_runner_result(result)
177 178 179 180 181 182 183 184

def test_evaluate():
    dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
    test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
    dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
    test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
    runner = CliRunner()
    with runner.isolated_filesystem():
185
        result = runner.invoke(commands.evaluate, [dev1])
186
        assert_click_runner_result(result)
187 188 189

    with runner.isolated_filesystem():
        result = runner.invoke(
190
            commands.evaluate, ['--output', 'my_plots.pdf',
191
                                '-n', 300, dev1, dev2])
192
        assert_click_runner_result(result)
193 194 195

    with runner.isolated_filesystem():
        result = runner.invoke(
196
            commands.evaluate, ['-e', dev1, test1, dev2, test2])
197
        assert_click_runner_result(result)