diff --git a/beat/core/agent.py b/beat/core/agent.py
index 5f90f8c130612e46fdc0003f0c01568bf9d83c53..6b2b2898c05912118dad2befabcf5e201c042453 100755
--- a/beat/core/agent.py
+++ b/beat/core/agent.py
@@ -28,6 +28,7 @@
import os
import shutil
+import simplejson
import logging
logger = logging.getLogger(__name__)
@@ -221,17 +222,43 @@ class Agent(object):
if db_address is not None:
configuration.dump_databases_provider_configuration(self.db_tempdir)
+ # Modify the paths to the databases in the dumped configuration files
+ root_folder = os.path.join(self.db_tempdir, 'prefix', 'databases')
+
+ database_paths = {}
+
+ for db_name in configuration.databases.keys():
+ json_path = os.path.join(root_folder, db_name + '.json')
+
+ with open(json_path, 'r') as f:
+ db_data = simplejson.load(f)
+
+ database_paths[db_name] = db_data['root_folder']
+ db_data['root_folder'] = os.path.join('/databases', db_name)
+
+ with open(json_path, 'w') as f:
+ simplejson.dump(db_data, f, indent=4)
+
# Server for our single client
self.server = Server(configuration.input_list, configuration.output_list,
host.ip)
- # Figures out the image to use
+ # Figures out the images to use
envkey = '%(name)s (%(version)s)' % configuration.data['environment']
if envkey not in host:
raise RuntimeError("Environment `%s' is not available on docker " \
"host `%s' - available environments are %s" % (envkey, host,
", ".join(host.environments.keys())))
+ if db_address is not None:
+ try:
+ db_envkey = host.db2docker(database_paths.keys())
+ except:
+ raise RuntimeError("No environment found for the databases `%s' " \
+ "- available environments are %s" % (
+ ", ".join(database_paths.keys()),
+ ", ".join(host.db_environments.keys())))
+
# Launches the process (0MQ client)
tmp_dir = os.path.join('/tmp', os.path.basename(self.tempdir))
cmd = ['execute', self.server.address, tmp_dir]
@@ -246,15 +273,23 @@ class Agent(object):
else:
if db_address is not None:
tmp_dir = os.path.join('/tmp', os.path.basename(self.db_tempdir))
- # db_cmd = ['bash', '-c', 'source activate beat_env; databases_provider %s %s' % (db_address, tmp_dir)]
- db_cmd = ['bash', '-c', 'databases_provider %s %s' % (db_address, tmp_dir)]
+ db_cmd = ['databases_provider', db_address, tmp_dir]
+
+ volumes = {}
+
+ for db_name, db_path in database_paths.items():
+ volumes[db_path] = {
+ 'bind': os.path.join('/databases', db_name),
+ 'mode': 'ro',
+ }
+ # Note: we only support one databases image loaded at the same time
self.db_process = dock.Popen(
host,
- 'docker.idiap.ch/beat/beat.env.db.examples:1.0.0',
- # 'docker.idiap.ch/beat/beat.env.db:1.0.0',
+ db_envkey,
command=db_cmd,
tmp_archive=self.db_tempdir,
+ volumes=volumes
)
self.process = dock.Popen(
diff --git a/beat/core/dock.py b/beat/core/dock.py
index 37ecfd76ece247352e3c4440d1d0693303664be1..9a2706703de3c9e809b7801105eb8ea504987528 100755
--- a/beat/core/dock.py
+++ b/beat/core/dock.py
@@ -62,17 +62,18 @@ class Host(object):
self.kwargs = kwargs
self.environments = {}
+ self.db_environments = {}
def setup(self, raise_on_errors=True):
self.client = docker.Client(**self.kwargs)
- self.environments = self._discover_environments(raise_on_errors)
+ (self.environments, self.db_environments) = self._discover_environments(raise_on_errors)
def __contains__(self, key):
- return key in self.environments
+ return (key in self.environments) or (key in self.db_environments)
def __str__(self):
@@ -84,13 +85,29 @@ class Host(object):
attrs = self.environments[key]
- if attrs['tag'] is not None: return attrs['tag']
+ if attrs['tag'] is not None:
+ return attrs['tag']
+
return attrs['short_id']
- def teardown(self):
+ def db2docker(self, db_names):
+ '''Returns a nice docker image name given a database name'''
+
+ def _all_in(db_names, databases):
+ return len([ x for x in db_names if x in databases ]) == len(db_names)
+
+ attrs = [ x for x in self.db_environments.values() if _all_in(db_names, x['databases']) ][0]
- for container in self.containers: self.rm(container)
+ if attrs['tag'] is not None:
+ return attrs['tag']
+
+ return attrs['short_id']
+
+
+ def teardown(self):
+ for container in self.containers:
+ self.rm(container)
def __enter__(self):
@@ -140,98 +157,118 @@ class Host(object):
"`describe' output cannot be parsed: %s", image, str(e))
return {}
- retval = {}
-
- for image in self.client.images():
- # call the "describe" application on each existing image with "beat" in
- # its name
- tag = image['RepoTags'][0] if image['RepoTags'] else None
- if (tag is None) or (tag.find('beat') == -1):
- continue
-
- id = image['Id'].split(':')[1][:12]
- logger.debug("Checking image `%s' (%s)...", tag, id)
- description = _describe(tag or id)
- if not description: continue
-
- key = description['name'] + ' (' + description['version'] + ')'
-
- if key in retval:
+ def _must_replace(image_tag, environments, key):
# this check avoids we do a new environment and, by mistake, override
# it with a previous version or the contrary.
if raise_on_errors:
raise RuntimeError("Environments at `%s' and `%s' have the " \
"same name (`%s'). Distinct environments must be " \
"uniquely named. Fix this and re-start." % \
- (tag or id, retval[key]['image'], key))
- else:
- new_version = None
- previous_version = None
+ (image_tag, environments[key]['image'], key))
+
+ new_version = None
+ previous_version = None
- parts = tag.split('/')
+ parts = image_tag.split('/')
+ if len(parts) > 1:
+ parts = parts[-1].split(':')
if len(parts) > 1:
- parts = parts[-1].split(':')
- if len(parts) > 1:
- new_version = parts[-1]
+ new_version = parts[-1]
- parts = retval[key]['tag'].split('/')
+ parts = environments[key]['tag'].split('/')
+ if len(parts) > 1:
+ parts = parts[-1].split(':')
if len(parts) > 1:
- parts = parts[-1].split(':')
- if len(parts) > 1:
- previous_version = parts[-1]
+ previous_version = parts[-1]
- replacement = False
- keep = False
+ replacement = False
+ keep = False
- if (new_version is not None) and (previous_version is not None):
- if new_version == 'latest':
- replacement = True
- elif previous_version == 'latest':
- keep = True
- else:
- try:
- new_version = tuple([ int(x) for x in new_version.split('.') ])
+ if (new_version is not None) and (previous_version is not None):
+ if new_version == 'latest':
+ replacement = True
+ elif previous_version == 'latest':
+ keep = True
+ else:
+ try:
+ new_version = tuple([ int(x) for x in new_version.split('.') ])
- try:
- previous_version = tuple([ int(x) for x in previous_version.split('.') ])
+ try:
+ previous_version = tuple([ int(x) for x in previous_version.split('.') ])
- if new_version > previous_version:
- replacement = True
- else:
- keep = True
- except:
+ if new_version > previous_version:
replacement = True
-
+ else:
+ keep = True
except:
- keep = True
+ replacement = True
- elif new_version is not None:
- replacement = True
- elif previous_version is not None:
- keep = True
+ except:
+ keep = True
+
+ elif new_version is not None:
+ replacement = True
+ elif previous_version is not None:
+ keep = True
+
+ if replacement:
+ logger.debug("Overriding **existing** environment `%s' in image `%s'", key, environments[key]['tag'])
+ elif keep:
+ logger.debug("Environment `%s' already existing in image `%s', we'll keep it", key, environments[key]['tag'])
+ return False
+ else:
+ logger.warn("Overriding **existing** environment `%s' image " \
+ "with `%s'. To avoid this warning make " \
+ "sure your docker images do not contain environments " \
+ "with the same names", key, environments[key]['image'])
+
+ return True
+
+
+ environments = {}
+ db_environments = {}
+
+ for image in self.client.images():
+ # call the "describe" application on each existing image with "beat" in
+ # its name
+ tag = image['RepoTags'][0] if image['RepoTags'] else None
+ if (tag is None) or (tag.find('beat') == -1):
+ continue
+
+ id = image['Id'].split(':')[1][:12]
+ logger.debug("Checking image `%s' (%s)...", tag, id)
+ description = _describe(tag or id)
+ if not description:
+ continue
+
+ key = description['name'] + ' (' + description['version'] + ')'
+
+ if description.has_key('databases'):
+ if (key in db_environments) and not _must_replace(tag, db_environments, key):
+ continue
+
+ db_environments[key] = description
+ db_environments[key]['image'] = image['Id']
+ db_environments[key]['tag'] = tag
+ db_environments[key]['short_id'] = id
+ db_environments[key]['nickname'] = tag or id
+ else:
+ if (key in environments) and not _must_replace(tag, environments, key):
+ continue
+
+ environments[key] = description
+ environments[key]['image'] = image['Id']
+ environments[key]['tag'] = tag
+ environments[key]['short_id'] = id
+ environments[key]['nickname'] = tag or id
- if replacement:
- logger.debug("Overriding **existing** environment `%s' in image `%s'", key, retval[key]['tag'])
- elif keep:
- logger.debug("Environment `%s' already existing in image `%s', we'll keep it", key, retval[key]['tag'])
- continue
- else:
- logger.warn("Overriding **existing** environment `%s' image " \
- "with `%s' (it was `%s). To avoid this warning make " \
- "sure your docker images do not contain environments " \
- "with the same names", key, retval[key]['image'],
- image['Id'])
-
- retval[key] = description
- retval[key]['image'] = image['Id']
- retval[key]['tag'] = tag
- retval[key]['short_id'] = id
- retval[key]['nickname'] = tag or id
logger.info("Registered `%s' -> `%s (%s)'", key, tag, id)
- logger.debug("Found %d environments", len(retval))
- return retval
+ logger.debug("Found %d environments and %d database environments", len(environments),
+ len(db_environments))
+
+ return (environments, db_environments)
def put_path(self, container, src, dest='/tmp', chmod=None):