Commit 6e65602f authored by Philip ABBET's avatar Philip ABBET
Browse files

Improve the database container handling

parent ca1dc069
......@@ -28,6 +28,7 @@
import os
import shutil
import simplejson
import logging
logger = logging.getLogger(__name__)
......@@ -221,17 +222,43 @@ class Agent(object):
if db_address is not None:
configuration.dump_databases_provider_configuration(self.db_tempdir)
# Modify the paths to the databases in the dumped configuration files
root_folder = os.path.join(self.db_tempdir, 'prefix', 'databases')
database_paths = {}
for db_name in configuration.databases.keys():
json_path = os.path.join(root_folder, db_name + '.json')
with open(json_path, 'r') as f:
db_data = simplejson.load(f)
database_paths[db_name] = db_data['root_folder']
db_data['root_folder'] = os.path.join('/databases', db_name)
with open(json_path, 'w') as f:
simplejson.dump(db_data, f, indent=4)
# Server for our single client
self.server = Server(configuration.input_list, configuration.output_list,
host.ip)
# Figures out the image to use
# Figures out the images to use
envkey = '%(name)s (%(version)s)' % configuration.data['environment']
if envkey not in host:
raise RuntimeError("Environment `%s' is not available on docker " \
"host `%s' - available environments are %s" % (envkey, host,
", ".join(host.environments.keys())))
if db_address is not None:
try:
db_envkey = host.db2docker(database_paths.keys())
except:
raise RuntimeError("No environment found for the databases `%s' " \
"- available environments are %s" % (
", ".join(database_paths.keys()),
", ".join(host.db_environments.keys())))
# Launches the process (0MQ client)
tmp_dir = os.path.join('/tmp', os.path.basename(self.tempdir))
cmd = ['execute', self.server.address, tmp_dir]
......@@ -246,15 +273,23 @@ class Agent(object):
else:
if db_address is not None:
tmp_dir = os.path.join('/tmp', os.path.basename(self.db_tempdir))
# db_cmd = ['bash', '-c', 'source activate beat_env; databases_provider %s %s' % (db_address, tmp_dir)]
db_cmd = ['bash', '-c', 'databases_provider %s %s' % (db_address, tmp_dir)]
db_cmd = ['databases_provider', db_address, tmp_dir]
volumes = {}
for db_name, db_path in database_paths.items():
volumes[db_path] = {
'bind': os.path.join('/databases', db_name),
'mode': 'ro',
}
# Note: we only support one databases image loaded at the same time
self.db_process = dock.Popen(
host,
'docker.idiap.ch/beat/beat.env.db.examples:1.0.0',
# 'docker.idiap.ch/beat/beat.env.db:1.0.0',
db_envkey,
command=db_cmd,
tmp_archive=self.db_tempdir,
volumes=volumes
)
self.process = dock.Popen(
......
......@@ -62,17 +62,18 @@ class Host(object):
self.kwargs = kwargs
self.environments = {}
self.db_environments = {}
def setup(self, raise_on_errors=True):
self.client = docker.Client(**self.kwargs)
self.environments = self._discover_environments(raise_on_errors)
(self.environments, self.db_environments) = self._discover_environments(raise_on_errors)
def __contains__(self, key):
return key in self.environments
return (key in self.environments) or (key in self.db_environments)
def __str__(self):
......@@ -84,13 +85,29 @@ class Host(object):
attrs = self.environments[key]
if attrs['tag'] is not None: return attrs['tag']
if attrs['tag'] is not None:
return attrs['tag']
return attrs['short_id']
def teardown(self):
def db2docker(self, db_names):
'''Returns a nice docker image name given a database name'''
def _all_in(db_names, databases):
return len([ x for x in db_names if x in databases ]) == len(db_names)
attrs = [ x for x in self.db_environments.values() if _all_in(db_names, x['databases']) ][0]
for container in self.containers: self.rm(container)
if attrs['tag'] is not None:
return attrs['tag']
return attrs['short_id']
def teardown(self):
for container in self.containers:
self.rm(container)
def __enter__(self):
......@@ -140,98 +157,118 @@ class Host(object):
"`describe' output cannot be parsed: %s", image, str(e))
return {}
retval = {}
for image in self.client.images():
# call the "describe" application on each existing image with "beat" in
# its name
tag = image['RepoTags'][0] if image['RepoTags'] else None
if (tag is None) or (tag.find('beat') == -1):
continue
id = image['Id'].split(':')[1][:12]
logger.debug("Checking image `%s' (%s)...", tag, id)
description = _describe(tag or id)
if not description: continue
key = description['name'] + ' (' + description['version'] + ')'
if key in retval:
def _must_replace(image_tag, environments, key):
# this check avoids we do a new environment and, by mistake, override
# it with a previous version or the contrary.
if raise_on_errors:
raise RuntimeError("Environments at `%s' and `%s' have the " \
"same name (`%s'). Distinct environments must be " \
"uniquely named. Fix this and re-start." % \
(tag or id, retval[key]['image'], key))
else:
new_version = None
previous_version = None
(image_tag, environments[key]['image'], key))
new_version = None
previous_version = None
parts = tag.split('/')
parts = image_tag.split('/')
if len(parts) > 1:
parts = parts[-1].split(':')
if len(parts) > 1:
parts = parts[-1].split(':')
if len(parts) > 1:
new_version = parts[-1]
new_version = parts[-1]
parts = retval[key]['tag'].split('/')
parts = environments[key]['tag'].split('/')
if len(parts) > 1:
parts = parts[-1].split(':')
if len(parts) > 1:
parts = parts[-1].split(':')
if len(parts) > 1:
previous_version = parts[-1]
previous_version = parts[-1]
replacement = False
keep = False
replacement = False
keep = False
if (new_version is not None) and (previous_version is not None):
if new_version == 'latest':
replacement = True
elif previous_version == 'latest':
keep = True
else:
try:
new_version = tuple([ int(x) for x in new_version.split('.') ])
if (new_version is not None) and (previous_version is not None):
if new_version == 'latest':
replacement = True
elif previous_version == 'latest':
keep = True
else:
try:
new_version = tuple([ int(x) for x in new_version.split('.') ])
try:
previous_version = tuple([ int(x) for x in previous_version.split('.') ])
try:
previous_version = tuple([ int(x) for x in previous_version.split('.') ])
if new_version > previous_version:
replacement = True
else:
keep = True
except:
if new_version > previous_version:
replacement = True
else:
keep = True
except:
keep = True
replacement = True
elif new_version is not None:
replacement = True
elif previous_version is not None:
keep = True
except:
keep = True
elif new_version is not None:
replacement = True
elif previous_version is not None:
keep = True
if replacement:
logger.debug("Overriding **existing** environment `%s' in image `%s'", key, environments[key]['tag'])
elif keep:
logger.debug("Environment `%s' already existing in image `%s', we'll keep it", key, environments[key]['tag'])
return False
else:
logger.warn("Overriding **existing** environment `%s' image " \
"with `%s'. To avoid this warning make " \
"sure your docker images do not contain environments " \
"with the same names", key, environments[key]['image'])
return True
environments = {}
db_environments = {}
for image in self.client.images():
# call the "describe" application on each existing image with "beat" in
# its name
tag = image['RepoTags'][0] if image['RepoTags'] else None
if (tag is None) or (tag.find('beat') == -1):
continue
id = image['Id'].split(':')[1][:12]
logger.debug("Checking image `%s' (%s)...", tag, id)
description = _describe(tag or id)
if not description:
continue
key = description['name'] + ' (' + description['version'] + ')'
if description.has_key('databases'):
if (key in db_environments) and not _must_replace(tag, db_environments, key):
continue
db_environments[key] = description
db_environments[key]['image'] = image['Id']
db_environments[key]['tag'] = tag
db_environments[key]['short_id'] = id
db_environments[key]['nickname'] = tag or id
else:
if (key in environments) and not _must_replace(tag, environments, key):
continue
environments[key] = description
environments[key]['image'] = image['Id']
environments[key]['tag'] = tag
environments[key]['short_id'] = id
environments[key]['nickname'] = tag or id
if replacement:
logger.debug("Overriding **existing** environment `%s' in image `%s'", key, retval[key]['tag'])
elif keep:
logger.debug("Environment `%s' already existing in image `%s', we'll keep it", key, retval[key]['tag'])
continue
else:
logger.warn("Overriding **existing** environment `%s' image " \
"with `%s' (it was `%s). To avoid this warning make " \
"sure your docker images do not contain environments " \
"with the same names", key, retval[key]['image'],
image['Id'])
retval[key] = description
retval[key]['image'] = image['Id']
retval[key]['tag'] = tag
retval[key]['short_id'] = id
retval[key]['nickname'] = tag or id
logger.info("Registered `%s' -> `%s (%s)'", key, tag, id)
logger.debug("Found %d environments", len(retval))
return retval
logger.debug("Found %d environments and %d database environments", len(environments),
len(db_environments))
return (environments, db_environments)
def put_path(self, container, src, dest='/tmp', chmod=None):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment