Lowest validation loss

Fixes #2 (closed)

When the lowest validation loss is being initialized during training, it will now check if the model is being loaded from a checkpoint and if there is a training log available. If available, it will find the lowest validation loss and initialize the lowest validation loss with that value from the training log before starting training.

src/ptbench/engine/trainer.py

@@ -65,6 +65,45 @@ def check_gpu(device):
         ), f"Device set to '{device}', but nvidia-smi is not installed"
 
 
+def initialize_lowest_validation_loss(logfile_name, arguments):
+    """Initialize the lowest validation loss from the logfile if it exists and
+    if the training does not start from epoch 0, which means that a previous
+    training session is resumed.
+
+    Parameters
+    ----------
+
+    logfile_name : str
+        The logfile_name which is a join between the output_folder and trainlog.csv
+
+    arguments : dict
+        start and end epochs
+    """
+
+    if arguments["epoch"] != 0 and os.path.exists(logfile_name):
+        # Open the CSV file
+        with open(logfile_name) as file:
+            reader = csv.DictReader(file)
+            column_name = "validation_loss"
+
+            if column_name not in reader.fieldnames:
+                return sys.float_info.max
+
+            # Get the values of the desired column as a list
+            values = [float(row[column_name]) for row in reader]
+
+            if not values:
+                return sys.float_info.max
+
+        lowest_value = min(values)
+        logger.info(
+            f"Found lowest validation loss from previous session: {lowest_value}"
+        )
+        return lowest_value
+
+    return sys.float_info.max
+
+
 def save_model_summary(output_folder, model):
     """Save a little summary of the model in a txt file.
 
@@ -569,7 +608,9 @@ def run(
 
     # the lowest validation loss obtained so far - this value is updated only
     # if a validation set is available
-    lowest_validation_loss = sys.float_info.max
+    lowest_validation_loss = initialize_lowest_validation_loss(
+        logfile_name, arguments
+    )
 
     # set a specific validation criterion if the user has set one
     criterion_valid = criterion_valid or criterion