Commit bce3b1d0 authored by Yannick DAYER's avatar Yannick DAYER

Merge branch 'minor' into 'master'

Some minor updates on the checkpoint wrapper and SGE

See merge request !67
parents 56847997 57015195
Pipeline #52217 passed with stages
in 19 minutes and 1 second
......@@ -9,7 +9,7 @@ QUEUE_DEFAULT = {
"memory": "8GB",
"io_big": False,
"resource_spec": "",
"max_jobs": 96,
"max_jobs": 128,
"resources": {"default": 1},
},
"q_1week": {
......
......@@ -267,7 +267,7 @@ class CheckpointWrapper(BaseWrapper, TransformerMixin):
load_func=None,
sample_attribute="data",
hash_fn=None,
attempts=5,
attempts=10,
**kwargs,
):
super().__init__(**kwargs)
......@@ -389,11 +389,12 @@ class CheckpointWrapper(BaseWrapper, TransformerMixin):
def save(self, sample):
path = self.make_path(sample)
os.makedirs(os.path.dirname(path), exist_ok=True)
# Gets sample.data or sample.<sample_attribute> if specified
to_save = getattr(sample, self.sample_attribute)
for _ in range(self.attempts):
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
self.save_func(to_save, path)
# test loading
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment