Skip to content
Snippets Groups Projects
Commit 0a0e9529 authored by Maxime DELITROZ's avatar Maxime DELITROZ
Browse files

[test/test_cli] updated unit tests for training, prediction and evaluation cli... 

[test/test_cli] updated unit tests for training, prediction and evaluation cli scripts. Removed radiological signs (rs) related tests
parent d606f37e
No related branches found
No related tags found
1 merge request!6Making use of LightningDataModule and simplification of data loading
Pipeline #77690 failed
Stage: qa
Stage: test
Stage: doc
Stage: dist
Hide whitespace changes
Inline Side-by-side
Showing
with 25 additions and 232 deletions
tests/test_cli.py
+ 25
− 232
View file @ 0a0e9529
...@@ -154,7 +154,6 @@ def test_evaluate_help(): ...@@ -154,7 +154,6 @@ def test_evaluate_help():
_check_help(evaluate) _check_help(evaluate)
@pytest.mark.skip(reason="Test need to be updated")
@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery") @pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
def test_train_pasa_montgomery(temporary_basedir): def test_train_pasa_montgomery(temporary_basedir):
from ptbench.scripts.train import train from ptbench.scripts.train import train
...@@ -191,7 +190,7 @@ def test_train_pasa_montgomery(temporary_basedir): ...@@ -191,7 +190,7 @@ def test_train_pasa_montgomery(temporary_basedir):
) )
== 1 == 1
) )
assert os.path.exists(os.path.join(output_folder, "model_summary.txt")) assert os.path.exists(os.path.join(output_folder, "model-summary.txt"))
keywords = { keywords = {
r"^Writing command-line for reproduction at .*$": 1, r"^Writing command-line for reproduction at .*$": 1,
...@@ -216,7 +215,6 @@ def test_train_pasa_montgomery(temporary_basedir): ...@@ -216,7 +215,6 @@ def test_train_pasa_montgomery(temporary_basedir):
) )
@pytest.mark.skip(reason="Test need to be updated")
@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery") @pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
def test_train_pasa_montgomery_from_checkpoint(temporary_basedir): def test_train_pasa_montgomery_from_checkpoint(temporary_basedir):
from ptbench.scripts.train import train from ptbench.scripts.train import train
...@@ -251,7 +249,7 @@ def test_train_pasa_montgomery_from_checkpoint(temporary_basedir): ...@@ -251,7 +249,7 @@ def test_train_pasa_montgomery_from_checkpoint(temporary_basedir):
== 1 == 1
) )
assert os.path.exists(os.path.join(output_folder, "model_summary.txt")) assert os.path.exists(os.path.join(output_folder, "model-summary.txt"))
with stdout_logging() as buf: with stdout_logging() as buf:
result = runner.invoke( result = runner.invoke(
...@@ -284,7 +282,7 @@ def test_train_pasa_montgomery_from_checkpoint(temporary_basedir): ...@@ -284,7 +282,7 @@ def test_train_pasa_montgomery_from_checkpoint(temporary_basedir):
== 2 == 2
) )
assert os.path.exists(os.path.join(output_folder, "model_summary.txt")) assert os.path.exists(os.path.join(output_folder, "model-summary.txt"))
keywords = { keywords = {
r"^Writing command-line for reproduction at .*$": 1, r"^Writing command-line for reproduction at .*$": 1,
...@@ -310,7 +308,7 @@ def test_train_pasa_montgomery_from_checkpoint(temporary_basedir): ...@@ -310,7 +308,7 @@ def test_train_pasa_montgomery_from_checkpoint(temporary_basedir):
) )
@pytest.mark.skip(reason="Test need to be updated") # @pytest.mark.skip(reason="Test need to be updated")
@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery") @pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
def test_predict_pasa_montgomery(temporary_basedir, datadir): def test_predict_pasa_montgomery(temporary_basedir, datadir):
from ptbench.scripts.predict import predict from ptbench.scripts.predict import predict
...@@ -318,7 +316,7 @@ def test_predict_pasa_montgomery(temporary_basedir, datadir): ...@@ -318,7 +316,7 @@ def test_predict_pasa_montgomery(temporary_basedir, datadir):
runner = CliRunner() runner = CliRunner()
with stdout_logging() as buf: with stdout_logging() as buf:
output_folder = str(temporary_basedir / "predictions") output = str(temporary_basedir / "predictions")
result = runner.invoke( result = runner.invoke(
predict, predict,
[ [
...@@ -326,29 +324,24 @@ def test_predict_pasa_montgomery(temporary_basedir, datadir): ...@@ -326,29 +324,24 @@ def test_predict_pasa_montgomery(temporary_basedir, datadir):
"montgomery", "montgomery",
"-vv", "-vv",
"--batch-size=1", "--batch-size=1",
f"--weight={str(datadir / 'lfs' / 'models' / 'pasa.ckpt')}", f"--weight={str(temporary_basedir / 'results' / 'model_final_epoch.ckpt')}",
f"--output-folder={output_folder}", f"--output={output}",
], ],
) )
_assert_exit_0(result) _assert_exit_0(result)
# check predictions are there assert os.path.exists(output)
train_predictions_file = os.path.join(output_folder, "train.csv")
validation_predictions_file = os.path.join(
output_folder, "validation.csv"
)
test_predictions_file = os.path.join(output_folder, "test.csv")
assert os.path.exists(train_predictions_file)
assert os.path.exists(validation_predictions_file)
assert os.path.exists(test_predictions_file)
keywords = { keywords = {
r"^Loading dataset: * without caching. Trade-off: CPU RAM: less | Disk: more$": 3,
r"^Loading checkpoint from .*$": 1,
r"^Restoring normalizer from checkpoint.$": 1, r"^Restoring normalizer from checkpoint.$": 1,
r"^Output folder: .*$": 1, r"^Running prediction on `train` split...$": 1,
r"^Loading dataset: * without caching. Trade-off: CPU RAM: less | Disk: more": 3, r"^Running prediction on `validation` split...$": 1,
r"^Saving predictions in .*$": 3, r"^Running prediction on `test` split...$": 1,
r"^Predictions saved to .*$": 1
} }
buf.seek(0) buf.seek(0)
logging_output = buf.read() logging_output = buf.read()
...@@ -360,7 +353,7 @@ def test_predict_pasa_montgomery(temporary_basedir, datadir): ...@@ -360,7 +353,7 @@ def test_predict_pasa_montgomery(temporary_basedir, datadir):
) )
@pytest.mark.skip(reason="Test need to be updated") # @pytest.mark.skip(reason="Test need to be updated")
@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery") @pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
def test_evaluate_pasa_montgomery(temporary_basedir): def test_evaluate_pasa_montgomery(temporary_basedir):
from ptbench.scripts.evaluate import evaluate from ptbench.scripts.evaluate import evaluate
...@@ -375,223 +368,23 @@ def test_evaluate_pasa_montgomery(temporary_basedir): ...@@ -375,223 +368,23 @@ def test_evaluate_pasa_montgomery(temporary_basedir):
[ [
"-vv", "-vv",
"montgomery", "montgomery",
f"--predictions-folder={prediction_folder}", f"--predictions={prediction_folder}",
f"--output-folder={output_folder}", f"--output-folder={output_folder}",
"--threshold=test", "--threshold=test"
"--steps=2000",
], ],
) )
_assert_exit_0(result) _assert_exit_0(result)
assert os.path.exists(os.path.join(output_folder, "scores.pdf"))
assert os.path.exists(os.path.join(output_folder, "plots.pdf")) assert os.path.exists(os.path.join(output_folder, "plots.pdf"))
assert os.path.exists(os.path.join(output_folder, "table.txt")) assert os.path.exists(os.path.join(output_folder, "summary.rst"))
keywords = {
r"^Evaluating threshold on.*$": 1,
r"^Maximum F1-score of.*$": 4,
r"^Set --f1_threshold=.*$": 1,
r"^Set --eer_threshold=.*$": 1,
}
buf.seek(0)
logging_output = buf.read()
for k, v in keywords.items():
assert _str_counter(k, logging_output) == v, (
f"Count for string '{k}' appeared "
f"({_str_counter(k, logging_output)}) "
f"instead of the expected {v}:\nOutput:\n{logging_output}"
)
@pytest.mark.skip(reason="Test need to be updated")
@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
def test_train_mlp_montgomery_rs(temporary_basedir, datadir):
from ptbench.scripts.train import train
runner = CliRunner()
with stdout_logging() as buf:
output_folder = str(temporary_basedir / "results/mlp")
result = runner.invoke(
train,
[
"mlp",
"montgomery_rs",
"-vv",
"--epochs=1",
"--batch-size=1",
f"--output-folder={output_folder}",
],
)
_assert_exit_0(result)
assert os.path.exists(
os.path.join(output_folder, "model_final_epoch.ckpt")
)
assert os.path.exists(
os.path.join(output_folder, "model_lowest_valid_loss.ckpt")
)
assert os.path.exists(os.path.join(output_folder, "constants.csv"))
assert os.path.exists(
os.path.join(output_folder, "logs_csv", "version_0", "metrics.csv")
)
assert os.path.exists(
os.path.join(output_folder, "logs_tensorboard", "version_0")
)
assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))
keywords = {
r"^Found \(dedicated\) '__train__' set for training$": 1,
r"^Found \(dedicated\) '__valid__' set for validation$": 1,
r"^Continuing from epoch 0$": 1,
r"^Saving model summary at.*$": 1,
}
buf.seek(0)
logging_output = buf.read()
for k, v in keywords.items():
assert _str_counter(k, logging_output) == v, (
f"Count for string '{k}' appeared "
f"({_str_counter(k, logging_output)}) "
f"instead of the expected {v}:\nOutput:\n{logging_output}"
)
@pytest.mark.skip(reason="Test need to be updated")
@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
def test_predict_mlp_montgomery_rs(temporary_basedir, datadir):
from ptbench.scripts.predict import predict
runner = CliRunner()
with stdout_logging() as buf:
output_folder = str(temporary_basedir / "predictions")
result = runner.invoke(
predict,
[
"mlp",
"montgomery_rs",
"-vv",
"--batch-size=1",
"--relevance-analysis",
f"--weight={str(datadir / 'lfs' / 'models' / 'mlp.ckpt')}",
f"--output-folder={output_folder}",
],
)
_assert_exit_0(result)
# check predictions are there
predictions_file = os.path.join(output_folder, "train/predictions.csv")
RA1 = os.path.join(output_folder, "train_RA.pdf")
RA2 = os.path.join(output_folder, "validation_RA.pdf")
RA3 = os.path.join(output_folder, "test_RA.pdf")
assert os.path.exists(predictions_file)
assert os.path.exists(RA1)
assert os.path.exists(RA2)
assert os.path.exists(RA3)
keywords = {
r"^Loading checkpoint from.*$": 1,
r"^Starting relevance analysis for subset.*$": 3,
r"^Creating and saving plot at.*$": 3,
}
buf.seek(0)
logging_output = buf.read()
for k, v in keywords.items():
assert _str_counter(k, logging_output) == v, (
f"Count for string '{k}' appeared "
f"({_str_counter(k, logging_output)}) "
f"instead of the expected {v}:\nOutput:\n{logging_output}"
)
@pytest.mark.skip(reason="Test need to be updated")
@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
def test_train_logreg_montgomery_rs(temporary_basedir, datadir):
from ptbench.scripts.train import train
runner = CliRunner()
with stdout_logging() as buf:
output_folder = str(temporary_basedir / "results/logreg")
result = runner.invoke(
train,
[
"logistic_regression",
"montgomery_rs",
"-vv",
"--epochs=1",
"--batch-size=1",
f"--output-folder={output_folder}",
],
)
_assert_exit_0(result)
assert os.path.exists(
os.path.join(output_folder, "model_final_epoch.ckpt")
)
assert os.path.exists(
os.path.join(output_folder, "model_lowest_valid_loss.ckpt")
)
assert os.path.exists(os.path.join(output_folder, "constants.csv"))
assert os.path.exists(
os.path.join(output_folder, "logs_csv", "version_0", "metrics.csv")
)
assert os.path.exists(
os.path.join(output_folder, "logs_tensorboard", "version_0")
)
assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))
keywords = {
r"^Found \(dedicated\) '__train__' set for training$": 1,
r"^Found \(dedicated\) '__valid__' set for validation$": 1,
r"^Continuing from epoch 0$": 1,
r"^Saving model summary at.*$": 1,
}
buf.seek(0)
logging_output = buf.read()
for k, v in keywords.items():
assert _str_counter(k, logging_output) == v, (
f"Count for string '{k}' appeared "
f"({_str_counter(k, logging_output)}) "
f"instead of the expected {v}:\nOutput:\n{logging_output}"
)
@pytest.mark.skip(reason="Test need to be updated")
@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
def test_predict_logreg_montgomery_rs(temporary_basedir, datadir):
from ptbench.scripts.predict import predict
runner = CliRunner()
with stdout_logging() as buf:
output_folder = str(temporary_basedir / "predictions")
result = runner.invoke(
predict,
[
"logistic_regression",
"montgomery_rs",
"-vv",
"--batch-size=1",
f"--weight={str(datadir / 'lfs' / 'models' / 'logreg.ckpt')}",
f"--output-folder={output_folder}",
],
)
_assert_exit_0(result)
# check predictions are there
predictions_file = os.path.join(output_folder, "train/predictions.csv")
wfile = os.path.join(output_folder, "LogReg_Weights.pdf")
assert os.path.exists(predictions_file)
assert os.path.exists(wfile)
keywords = { keywords = {
r"^Loading checkpoint from.*$": 1, r"^Setting --threshold=.*$": 1,
r"^Logistic regression identified: saving model weights.*$": 1, r"^Analyzing split `train`...$": 1,
r"^Analyzing split `validation`...$": 1,
r"^Analyzing split `test`...$": 1,
r"^Saving measures at .*$": 1,
r"^Saving figures at .*$": 1
} }
buf.seek(0) buf.seek(0)
logging_output = buf.read() logging_output = buf.read()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment