[experiments] Index experiment database(s) if not already done before running

beat/cmdline/experiments.py

@@ -74,8 +74,6 @@ Options:
 import os
 import logging
 import glob
-logger = logging.getLogger(__name__)
-
 import oset
 import simplejson
 
@@ -87,6 +85,11 @@ from beat.core.execution import LocalExecutor
 from beat.core.utils import NumpyJSONEncoder
 from beat.core.data import CachedDataSource, load_data_index
 from beat.core.dock import Host
+from beat.core.hash import toPath
+from beat.core.hash import hashDataset
+
+
+logger = logging.getLogger(__name__)
 
 
 def run_experiment(configuration, name, force, use_docker, use_local):
@@ -144,6 +147,17 @@ def run_experiment(configuration, name, force, use_docker, use_local):
     elif s <= tb: return "%.2f gigabytes" % (s/gb)
     return "%.2f terabytes" % (s/tb)
 
+
+  def index_experiment_databases(prefix, experiment):
+    for block_name, infos in experiment.datasets.items():
+      view = infos['database'].view(infos['protocol'], infos['set'])
+      filename = toPath(hashDataset(infos['database'].name,
+                                    infos['protocol'],
+                                    infos['set']),
+                        suffix='.db')
+      if not os.path.exists(filename):
+        view.index(os.path.join(prefix, filename))
+
   dataformat_cache = {}
   database_cache = {}
   algorithm_cache = {}
@@ -161,6 +175,9 @@ def run_experiment(configuration, name, force, use_docker, use_local):
     os.makedirs(configuration.cache)
     logger.info("Created cache path `%s'", configuration.cache)
 
+
+  index_experiment_databases(configuration.path, experiment)
+
   scheduled = experiment.setup()
 
   if use_docker: