From b28ff7d1a8fe5016a0ba027221beb8936e74a50f Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Thu, 23 Apr 2020 11:03:04 +0200
Subject: [PATCH] [test.test_cli] Test more variants; Use regexp for string
matching everywhere
---
bob/ip/binseg/test/test_cli.py | 147 ++++++++++++++++++++-------------
1 file changed, 89 insertions(+), 58 deletions(-)
diff --git a/bob/ip/binseg/test/test_cli.py b/bob/ip/binseg/test/test_cli.py
index 89795095..e39fba54 100644
--- a/bob/ip/binseg/test/test_cli.py
+++ b/bob/ip/binseg/test/test_cli.py
@@ -64,11 +64,10 @@ def test_experiment_help():
def _str_counter(substr, s):
- return sum(1 for _ in re.finditer(r"%s" % re.escape(substr), s))
+ return sum(1 for _ in re.finditer(substr, s, re.MULTILINE))
-@rc_variable_set("bob.ip.binseg.stare.datadir")
-def test_experiment_stare():
+def _check_experiment_stare(overlay):
from ..script.experiment import experiment
@@ -88,18 +87,17 @@ def test_experiment_stare():
config.flush()
output_folder = "results"
- result = runner.invoke(
- experiment,
- [
+ options = [
"m2unet",
config.name,
"-vv",
"--epochs=1",
"--batch-size=1",
- "--overlayed",
f"--output-folder={output_folder}",
- ],
- )
+ ]
+ if overlay:
+ options += ["--overlayed"]
+ result = runner.invoke(experiment, options)
_assert_exit_0(result)
# check command-line
@@ -118,55 +116,68 @@ def test_experiment_stare():
assert os.path.exists(basedir)
nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.hdf5")), 20)
- # check overlayed images are there (since we requested them)
overlay_folder = os.path.join(output_folder, "overlayed", "predictions")
basedir = os.path.join(overlay_folder, "stare-images")
- assert os.path.exists(basedir)
- nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 20)
+ if overlay:
+ # check overlayed images are there (since we requested them)
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 20)
+ else:
+ assert not os.path.exists(basedir)
# check evaluation outputs
eval_folder = os.path.join(output_folder, "analysis")
second_folder = os.path.join(eval_folder, "second-annotator")
assert os.path.exists(os.path.join(eval_folder, "train", "metrics.csv"))
assert os.path.exists(os.path.join(eval_folder, "test", "metrics.csv"))
- assert os.path.exists(os.path.join(second_folder, "train", "metrics.csv"))
- assert os.path.exists(os.path.join(second_folder, "test", "metrics.csv"))
+ assert os.path.exists(
+ os.path.join(second_folder, "train", "metrics.csv")
+ )
+ assert os.path.exists(
+ os.path.join(second_folder, "test", "metrics.csv")
+ )
- # check overlayed images are there (since we requested them)
overlay_folder = os.path.join(output_folder, "overlayed", "analysis")
basedir = os.path.join(overlay_folder, "stare-images")
- assert os.path.exists(basedir)
- nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 20)
-
- # check overlayed images from first-to-second annotator comparisons are
- # there (since we requested them)
+ if overlay:
+ # check overlayed images are there (since we requested them)
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 20)
+ else:
+ assert not os.path.exists(basedir)
+
+ # check overlayed images from first-to-second annotator comparisons
+ # are there (since we requested them)
overlay_folder = os.path.join(output_folder, "overlayed", "analysis",
"second-annotator")
basedir = os.path.join(overlay_folder, "stare-images")
- assert os.path.exists(basedir)
- nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 20)
+ if overlay:
+ assert os.path.exists(basedir)
+ nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 20)
+ else:
+ assert not os.path.exists(basedir)
# check outcomes of the comparison phase
assert os.path.exists(os.path.join(output_folder, "comparison.pdf"))
- keywords = { # from different logging systems
- "Started training": 1, # logging
- "Found (dedicated) '__train__' set for training": 1, # logging
- "epoch: 1|total-time": 1, # logging
- "Saving checkpoint": 1, # logging
- "Ended training": 1, # logging
- "Started prediction": 1, # logging
- "Loading checkpoint from": 2, # logging
- # "Saving results/overlayed/probabilities": 1, #tqdm.write
- "Ended prediction": 1, # logging
- "Started evaluation": 1, # logging
- "Highest F1-score of": 4, # logging
- "Saving overall precision-recall plot": 2, # logging
- # "Saving results/overlayed/analysis": 1, #tqdm.write
- "Ended evaluation": 1, # logging
- "Started comparison": 1, # logging
- "Loading metrics from": 4, # logging
- "Ended comparison": 1, # logging
+ keywords = {
+ r"^Started training$": 1,
+ r"^Found \(dedicated\) '__train__' set for training$": 1,
+ r"^epoch: \d+\|total-time": 1,
+ r"^Saving checkpoint": 1,
+ r"^Ended training$": 1,
+ r"^Started prediction$": 1,
+ r"^Loading checkpoint from": 2,
+ r"^Ended prediction$": 1,
+ r"^Started evaluation$": 1,
+ r"^Maximum F1-score of.*\(chosen \*a posteriori\*\)$": 3,
+ r"^F1-score of.*\(chosen \*a priori\*\)$": 2,
+ r"^Maximum F1-score of .* \(second annotator\)$": 2,
+ r"^Saving overall precision-recall plot at .*$": 2,
+ r"^Ended evaluation$": 1,
+ r"^Started comparison$": 1,
+ r"^Loading metrics from": 4,
+ r"^Ended comparison.*$": 1,
}
buf.seek(0)
logging_output = buf.read()
@@ -182,6 +193,16 @@ def test_experiment_stare():
)
+@rc_variable_set("bob.ip.binseg.stare.datadir")
+def test_experiment_stare_with_overlay():
+ _check_experiment_stare(overlay=True)
+
+
+@rc_variable_set("bob.ip.binseg.stare.datadir")
+def test_experiment_stare_without_overlay():
+ _check_experiment_stare(overlay=False)
+
+
def _check_train(runner):
from ..script.train import train
@@ -202,8 +223,14 @@ def _check_train(runner):
output_folder = "results"
result = runner.invoke(
train,
- ["m2unet", config.name, "-vv", "--epochs=1", "--batch-size=1",
- f"--output-folder={output_folder}"],
+ [
+ "m2unet",
+ config.name,
+ "-vv",
+ "--epochs=1",
+ "--batch-size=1",
+ f"--output-folder={output_folder}",
+ ],
)
_assert_exit_0(result)
@@ -211,11 +238,11 @@ def _check_train(runner):
assert os.path.exists(os.path.join(output_folder, "last_checkpoint"))
assert os.path.exists(os.path.join(output_folder, "trainlog.csv"))
- keywords = { # from different logging systems
- "Continuing from epoch 0": 1, # logging
- "epoch: 1|total-time": 1, # logging
- f"Saving checkpoint to {output_folder}/model_final.pth": 1, # logging
- "Total training time:": 1, # logging
+ keywords = {
+ r"^Continuing from epoch 0$": 1,
+ r"^epoch: \d+\|total-time": 1,
+ rf"^Saving checkpoint to {output_folder}/model_final.pth$": 1,
+ r"^Total training time:": 1,
}
buf.seek(0)
logging_output = buf.read()
@@ -276,9 +303,9 @@ def _check_predict(runner):
assert os.path.exists(basedir)
nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 10)
- keywords = { # from different logging systems
- "Loading checkpoint from": 1, # logging
- "Total time:": 1, # logging
+ keywords = {
+ r"^Loading checkpoint from.*$": 1,
+ r"^Total time:.*$": 1,
}
buf.seek(0)
logging_output = buf.read()
@@ -337,10 +364,12 @@ def _check_evaluate(runner):
assert os.path.exists(basedir)
nose.tools.eq_(len(fnmatch.filter(os.listdir(basedir), "*.png")), 10)
- keywords = { # from different logging systems
- "Skipping dataset '__train__'": 0, # logging
- "Saving averages over all input images": 2, # logging
- "Highest F1-score": 2, # logging
+ keywords = {
+ r"^Skipping dataset '__train__'": 0,
+ r"^Saving averages over all input images.*$": 2,
+ r"^Maximum F1-score of.*\(chosen \*a posteriori\*\)$": 1,
+ r"^F1-score of.*\(chosen \*a priori\*\)$": 1,
+ r"^Maximum F1-score of .* \(second annotator\)$": 1,
}
buf.seek(0)
logging_output = buf.read()
@@ -370,16 +399,18 @@ def _check_compare(runner):
[
"-vv",
# label - path to metrics
- "test", os.path.join(output_folder, "metrics.csv"),
- "test (2nd. human)", os.path.join(second_folder, "metrics.csv"),
+ "test",
+ os.path.join(output_folder, "metrics.csv"),
+ "test (2nd. human)",
+ os.path.join(second_folder, "metrics.csv"),
],
)
_assert_exit_0(result)
assert os.path.exists("comparison.pdf")
- keywords = { # from different logging systems
- "Loading metrics from": 2, # logging
+ keywords = {
+ r"^Loading metrics from": 2,
}
buf.seek(0)
logging_output = buf.read()
--
GitLab