Training stop in the middle due to "psutil.AccessDenied exception"
The error occurred in the middle of the training when using GPU.
Traceback (most recent call last):
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/psutil/_pslinux.py", line 1576, in wrapper
return fun(self, *args, **kwargs)
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/psutil/_pslinux.py", line 2089, in open_files
files = os.listdir("%s/%s/fd" % (self._procfs_path, self.pid))
PermissionError: [Errno 13] Permission denied: '/proc/6852/fd'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/idiap/temp/dkhalil/bob.ip.binseg/bin/bob", line 19, in <module>
sys.exit(bob.extension.scripts.main_cli())
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/remote/idiap.svm/temp.masterai/dkhalil/bob.ip.binseg/bob/ip/binseg/script/experiment.py", line 309, in experiment
ctx.invoke(
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/remote/idiap.svm/temp.masterai/dkhalil/bob.ip.binseg/bob/ip/binseg/script/train.py", line 302, in train
run(
File "/remote/idiap.svm/temp.masterai/dkhalil/bob.ip.binseg/bob/ip/binseg/engine/trainer.py", line 310, in run
logdata += cpu_log()
File "/remote/idiap.svm/temp.masterai/dkhalil/bob.ip.binseg/bob/ip/binseg/utils/resources.py", line 216, in cpu_log
("cpu_open_files", sum(len(k.open_files()) for k in _CLUSTER)),
File "/remote/idiap.svm/temp.masterai/dkhalil/bob.ip.binseg/bob/ip/binseg/utils/resources.py", line 216, in <genexpr>
("cpu_open_files", sum(len(k.open_files()) for k in _CLUSTER)),
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/psutil/__init__.py", line 1138, in open_files
return self._proc.open_files()
File "/idiap/temp/dkhalil/miniconda3/envs/dev/lib/python3.8/site-packages/psutil/_pslinux.py", line 1578, in wrapper
raise AccessDenied(self.pid, self._name)
psutil.AccessDenied: psutil.AccessDenied (pid=6852)