From c60bd4f0a4695adc58a566042291cd10b515582a Mon Sep 17 00:00:00 2001
From: dcarron <daniel.carron@idiap.ch>
Date: Wed, 12 Apr 2023 16:54:24 +0200
Subject: [PATCH] Updated tests
---
tests/test_cli.py | 84 +++++++++++++++++++++++--------------------
tests/test_summary.py | 5 +--
2 files changed, 48 insertions(+), 41 deletions(-)
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 2feb5e6c..796a795c 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -195,14 +195,18 @@ def test_train_pasa_montgomery(temporary_basedir):
_assert_exit_0(result)
assert os.path.exists(
- os.path.join(output_folder, "model_final_epoch.pth")
+ os.path.join(output_folder, "model_final_epoch.ckpt")
)
assert os.path.exists(
- os.path.join(output_folder, "model_lowest_valid_loss.pth")
+ os.path.join(output_folder, "model_lowest_valid_loss.ckpt")
)
- assert os.path.exists(os.path.join(output_folder, "last_checkpoint"))
assert os.path.exists(os.path.join(output_folder, "constants.csv"))
- assert os.path.exists(os.path.join(output_folder, "trainlog.csv"))
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_csv", "version_0", "metrics.csv")
+ )
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_tensorboard", "version_0")
+ )
assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))
keywords = {
@@ -210,10 +214,6 @@ def test_train_pasa_montgomery(temporary_basedir):
r"^Found \(dedicated\) '__valid__' set for validation$": 1,
r"^Continuing from epoch 0$": 1,
r"^Saving model summary at.*$": 1,
- r"^Model has.*$": 1,
- r"^Saving checkpoint": 2,
- r"^Total training time:": 1,
- r"^Z-normalization with mean": 1,
}
buf.seek(0)
logging_output = buf.read()
@@ -247,13 +247,17 @@ def test_train_pasa_montgomery_from_checkpoint(temporary_basedir):
)
_assert_exit_0(result0)
- assert os.path.exists(os.path.join(output_folder, "model_final_epoch.pth"))
+ assert os.path.exists(os.path.join(output_folder, "model_final_epoch.ckpt"))
assert os.path.exists(
- os.path.join(output_folder, "model_lowest_valid_loss.pth")
+ os.path.join(output_folder, "model_lowest_valid_loss.ckpt")
)
- assert os.path.exists(os.path.join(output_folder, "last_checkpoint"))
assert os.path.exists(os.path.join(output_folder, "constants.csv"))
- assert os.path.exists(os.path.join(output_folder, "trainlog.csv"))
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_csv", "version_0", "metrics.csv")
+ )
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_tensorboard", "version_0")
+ )
assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))
with stdout_logging() as buf:
@@ -272,25 +276,25 @@ def test_train_pasa_montgomery_from_checkpoint(temporary_basedir):
_assert_exit_0(result)
assert os.path.exists(
- os.path.join(output_folder, "model_final_epoch.pth")
+ os.path.join(output_folder, "model_final_epoch.ckpt")
)
assert os.path.exists(
- os.path.join(output_folder, "model_lowest_valid_loss.pth")
+ os.path.join(output_folder, "model_lowest_valid_loss.ckpt")
)
- assert os.path.exists(os.path.join(output_folder, "last_checkpoint"))
assert os.path.exists(os.path.join(output_folder, "constants.csv"))
- assert os.path.exists(os.path.join(output_folder, "trainlog.csv"))
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_csv", "version_0", "metrics.csv")
+ )
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_tensorboard", "version_0")
+ )
assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))
keywords = {
r"^Found \(dedicated\) '__train__' set for training$": 1,
r"^Found \(dedicated\) '__valid__' set for validation$": 1,
- r"^Continuing from epoch 1$": 1,
+ r"^Continuing from epoch 0$": 1,
r"^Saving model summary at.*$": 1,
- r"^Model has.*$": 1,
- r"^Found lowest validation loss from previous session.*$": 1,
- r"^Total training time:": 1,
- r"^Z-normalization with mean": 1,
}
buf.seek(0)
logging_output = buf.read()
@@ -302,10 +306,10 @@ def test_train_pasa_montgomery_from_checkpoint(temporary_basedir):
f"instead of the expected {v}:\nOutput:\n{logging_output}"
)
- extra_keyword = "Saving checkpoint"
- assert (
- extra_keyword in logging_output
- ), f"String '{extra_keyword}' did not appear at least once in the output:\nOutput:\n{logging_output}"
+ # extra_keyword = "Saving checkpoint"
+ # assert (
+ # extra_keyword in logging_output
+ # ), f"String '{extra_keyword}' did not appear at least once in the output:\nOutput:\n{logging_output}"
@pytest.mark.skip_if_rc_var_not_set("datadir.montgomery")
@@ -513,14 +517,18 @@ def test_train_signstotb_montgomery_rs(temporary_basedir, datadir):
_assert_exit_0(result)
assert os.path.exists(
- os.path.join(output_folder, "model_final_epoch.pth")
+ os.path.join(output_folder, "model_final_epoch.ckpt")
)
assert os.path.exists(
- os.path.join(output_folder, "model_lowest_valid_loss.pth")
+ os.path.join(output_folder, "model_lowest_valid_loss.ckpt")
)
- assert os.path.exists(os.path.join(output_folder, "last_checkpoint"))
assert os.path.exists(os.path.join(output_folder, "constants.csv"))
- assert os.path.exists(os.path.join(output_folder, "trainlog.csv"))
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_csv", "version_0", "metrics.csv")
+ )
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_tensorboard", "version_0")
+ )
assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))
keywords = {
@@ -528,9 +536,6 @@ def test_train_signstotb_montgomery_rs(temporary_basedir, datadir):
r"^Found \(dedicated\) '__valid__' set for validation$": 1,
r"^Continuing from epoch 0$": 1,
r"^Saving model summary at.*$": 1,
- r"^Model has.*$": 1,
- r"^Saving checkpoint": 2,
- r"^Total training time:": 1,
}
buf.seek(0)
logging_output = buf.read()
@@ -614,14 +619,18 @@ def test_train_logreg_montgomery_rs(temporary_basedir, datadir):
_assert_exit_0(result)
assert os.path.exists(
- os.path.join(output_folder, "model_final_epoch.pth")
+ os.path.join(output_folder, "model_final_epoch.ckpt")
)
assert os.path.exists(
- os.path.join(output_folder, "model_lowest_valid_loss.pth")
+ os.path.join(output_folder, "model_lowest_valid_loss.ckpt")
)
- assert os.path.exists(os.path.join(output_folder, "last_checkpoint"))
assert os.path.exists(os.path.join(output_folder, "constants.csv"))
- assert os.path.exists(os.path.join(output_folder, "trainlog.csv"))
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_csv", "version_0", "metrics.csv")
+ )
+ assert os.path.exists(
+ os.path.join(output_folder, "logs_tensorboard", "version_0")
+ )
assert os.path.exists(os.path.join(output_folder, "model_summary.txt"))
keywords = {
@@ -629,9 +638,6 @@ def test_train_logreg_montgomery_rs(temporary_basedir, datadir):
r"^Found \(dedicated\) '__valid__' set for validation$": 1,
r"^Continuing from epoch 0$": 1,
r"^Saving model summary at.*$": 1,
- r"^Model has.*$": 1,
- r"^Saving checkpoint": 2,
- r"^Total training time:": 1,
}
buf.seek(0)
logging_output = buf.read()
diff --git a/tests/test_summary.py b/tests/test_summary.py
index 4315a39f..1f178fdc 100644
--- a/tests/test_summary.py
+++ b/tests/test_summary.py
@@ -4,7 +4,8 @@
import unittest
-from ptbench.models.pasa import build_pasa
+import ptbench.configs.models.pasa as pasa_config
+
from ptbench.utils.summary import summary
@@ -12,7 +13,7 @@ class Tester(unittest.TestCase):
"""Unit test for model architectures."""
def test_summary_driu(self):
- model = build_pasa()
+ model = pasa_config.model
s, param = summary(model)
self.assertIsInstance(s, str)
self.assertIsInstance(param, int)
--
GitLab