Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.cmdline
Commits
ed9c1e86
Commit
ed9c1e86
authored
Mar 02, 2018
by
Samuel GAIST
Browse files
[experiments] Index experiment database(s) if not already done before running
parent
240ee9b8
Changes
1
Hide whitespace changes
Inline
Side-by-side
beat/cmdline/experiments.py
View file @
ed9c1e86
...
...
@@ -74,8 +74,6 @@ Options:
import
os
import
logging
import
glob
logger
=
logging
.
getLogger
(
__name__
)
import
oset
import
simplejson
...
...
@@ -87,6 +85,11 @@ from beat.core.execution import LocalExecutor
from
beat.core.utils
import
NumpyJSONEncoder
from
beat.core.data
import
CachedDataSource
,
load_data_index
from
beat.core.dock
import
Host
from
beat.core.hash
import
toPath
from
beat.core.hash
import
hashDataset
logger
=
logging
.
getLogger
(
__name__
)
def
run_experiment
(
configuration
,
name
,
force
,
use_docker
,
use_local
):
...
...
@@ -144,6 +147,17 @@ def run_experiment(configuration, name, force, use_docker, use_local):
elif
s
<=
tb
:
return
"%.2f gigabytes"
%
(
s
/
gb
)
return
"%.2f terabytes"
%
(
s
/
tb
)
def
index_experiment_databases
(
prefix
,
experiment
):
for
block_name
,
infos
in
experiment
.
datasets
.
items
():
view
=
infos
[
'database'
].
view
(
infos
[
'protocol'
],
infos
[
'set'
])
filename
=
toPath
(
hashDataset
(
infos
[
'database'
].
name
,
infos
[
'protocol'
],
infos
[
'set'
]),
suffix
=
'.db'
)
if
not
os
.
path
.
exists
(
filename
):
view
.
index
(
os
.
path
.
join
(
prefix
,
filename
))
dataformat_cache
=
{}
database_cache
=
{}
algorithm_cache
=
{}
...
...
@@ -161,6 +175,9 @@ def run_experiment(configuration, name, force, use_docker, use_local):
os
.
makedirs
(
configuration
.
cache
)
logger
.
info
(
"Created cache path `%s'"
,
configuration
.
cache
)
index_experiment_databases
(
configuration
.
path
,
experiment
)
scheduled
=
experiment
.
setup
()
if
use_docker
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment