Skip to content
Snippets Groups Projects
Commit e37b31d5 authored by Gokhan OZBULAK's avatar Gokhan OZBULAK
Browse files

Add more train info into experiment table and relocate trainlog.pdf path. #69

parent d354388c
No related branches found
No related tags found
1 merge request!41Add more train info into experiment table and relocate trainlog.pdf path. #69
Pipeline #87501 failed
...@@ -125,9 +125,9 @@ def upload( ...@@ -125,9 +125,9 @@ def upload(
) )
# get train files # get train files
train_log_file = experiment_folder / "trainlog.pdf"
train_folder = experiment_folder / "model" train_folder = experiment_folder / "model"
train_meta_file = train_folder / "meta.json" train_meta_file = train_folder / "meta.json"
train_log_file = train_folder / "trainlog.pdf"
train_model_file = get_checkpoint_to_run_inference(train_folder) train_model_file = get_checkpoint_to_run_inference(train_folder)
train_files = [train_meta_file, train_model_file, train_log_file] train_files = [train_meta_file, train_model_file, train_log_file]
...@@ -149,7 +149,6 @@ def upload( ...@@ -149,7 +149,6 @@ def upload(
f"permitted maximum ({upload_limit_mb:.2f} MB)." f"permitted maximum ({upload_limit_mb:.2f} MB)."
) )
# prepare experiment and run names
with train_meta_file.open("r") as meta_file: with train_meta_file.open("r") as meta_file:
train_data = json.load(meta_file) train_data = json.load(meta_file)
...@@ -157,6 +156,9 @@ def upload( ...@@ -157,6 +156,9 @@ def upload(
evaluation_data = json.load(meta_file) evaluation_data = json.load(meta_file)
evaluation_data = evaluation_data["test"] evaluation_data = evaluation_data["test"]
# get lowest validation epoch
best_epoch = str(train_model_file).split('.')[0].split('=')[1]
experiment_name = ( experiment_name = (
experiment_name experiment_name
or f"{train_data['model-name']}-{train_data['database-name']}" or f"{train_data['model-name']}-{train_data['database-name']}"
...@@ -177,6 +179,18 @@ def upload( ...@@ -177,6 +179,18 @@ def upload(
click.echo("Uploading metrics...") click.echo("Uploading metrics...")
for k in [
"epochs",
"batch-size",
]:
click.secho(f" -> `{k}` ({train_data[k]})")
mlflow.log_param(k, train_data[k])
click.secho(f" -> `#accumulations` ({train_data['batch-chunk-count']})")
mlflow.log_param("#Accumulations", train_data['batch-chunk-count'])
click.secho(f" -> `epoch (best)` ({best_epoch})")
mlflow.log_param("Epoch (best)", best_epoch)
for k in [ for k in [
"threshold", "threshold",
"precision", "precision",
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment