Commit ed9c1e86 authored by Samuel GAIST's avatar Samuel GAIST
Browse files

[experiments] Index experiment database(s) if not already done before running

parent 240ee9b8
......@@ -74,8 +74,6 @@ Options:
import os
import logging
import glob
logger = logging.getLogger(__name__)
import oset
import simplejson
......@@ -87,6 +85,11 @@ from beat.core.execution import LocalExecutor
from beat.core.utils import NumpyJSONEncoder
from beat.core.data import CachedDataSource, load_data_index
from beat.core.dock import Host
from beat.core.hash import toPath
from beat.core.hash import hashDataset
logger = logging.getLogger(__name__)
def run_experiment(configuration, name, force, use_docker, use_local):
......@@ -144,6 +147,17 @@ def run_experiment(configuration, name, force, use_docker, use_local):
elif s <= tb: return "%.2f gigabytes" % (s/gb)
return "%.2f terabytes" % (s/tb)
def index_experiment_databases(prefix, experiment):
for block_name, infos in experiment.datasets.items():
view = infos['database'].view(infos['protocol'], infos['set'])
filename = toPath(hashDataset(infos['database'].name,
infos['protocol'],
infos['set']),
suffix='.db')
if not os.path.exists(filename):
view.index(os.path.join(prefix, filename))
dataformat_cache = {}
database_cache = {}
algorithm_cache = {}
......@@ -161,6 +175,9 @@ def run_experiment(configuration, name, force, use_docker, use_local):
os.makedirs(configuration.cache)
logger.info("Created cache path `%s'", configuration.cache)
index_experiment_databases(configuration.path, experiment)
scheduled = experiment.setup()
if use_docker:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment