Commit 8cace2f8 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'handle-none-data' into 'master'

pickle VideoLikeContainers when data dtype is not supported by hdf5 format

Closes #18

See merge request !45
parents 3b4f0a95 d9e65a6f
Pipeline #53690 passed with stage
in 16 minutes and 38 seconds
import tempfile
import time
import numpy as np
from import is_library_available
from import datafile
from import reader
import numpy as np
regenerate_refs = False
......@@ -24,9 +26,7 @@ def test_video_as_array():
video = video[None, ...]
np.testing.assert_allclose(video, video_slice)
video =
path, max_number_of_frames=3
video =, max_number_of_frames=3)
assert len(video) == 3, len(video)
assert video.indices == [13, 41, 69], video.indices
assert video.shape == (3, 3, 480, 640), video.shape
......@@ -68,8 +68,14 @@ def test_video_like_container():
loaded_container =
assert container == loaded_container
np.testing.assert_equal(np.array(, np.array(
np.array(container.indices), np.array(loaded_container.indices)
# test saving and loading None arrays
with tempfile.NamedTemporaryFile(suffix=".pkl") as f:
data = [None] * 10 + [1]
indices = range(11)
frame_container =, indices)
loaded =
assert loaded == frame_container
import logging
import pickle
from import selected_indices
from import reader
import h5py
import numpy as np
from import selected_indices
from import reader
logger = logging.getLogger(__name__)
......@@ -226,21 +227,34 @@ class VideoLikeContainer:
def __array__(self, dtype=None, *args, **kwargs):
return np.asarray(, dtype, *args, **kwargs)
def __eq__(self, o: object) -> bool:
return np.array_equal(, and np.array_equal(
self.indices, o.indices
def save(self, file):
self.save_function(self, file)
def save_function(other, file):
with h5py.File(file, mode="w") as f:
f["data"] =
f["indices"] = other.indices
# revert to saving data in pickles when the dtype is not supported by hdf5
except TypeError:
with open(file, "wb") as f:
pickle.dump({"data":, "indices": other.indices}, f)
def load(cls, file):
# weak closing of the hdf5 file so we don't load all the data into
# memory
f = h5py.File(file, mode="r")
data = f["data"]
indices = f["indices"]
self = cls(data=data, indices=indices)
loaded = {"data": f["data"], "indices": f["indices"]}
except OSError:
with open(file, "rb") as f:
loaded = pickle.load(f)
self = cls(**loaded)
return self
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment