Commit a1ed95f3 authored by André Anjos's avatar André Anjos 💬

All tests are now passing with our new docker backend

parent abee7883
Pipeline #2459 failed with stage
......@@ -35,15 +35,28 @@ logger = logging.getLogger(__name__)
import gevent
import as zmq
import requests
from gevent import monkey
from . import utils
from . import async
from . import dock
from . import baseformat
def get_network_ip_address(host_address):
'''Returns the most sensible network IP address given a host address'''
import socket
import difflib
possible_addresses = socket.gethostbyname_ex(socket.gethostname())[2]
return difflib.get_close_matches(host_address, possible_addresses)[0]
class Server(gevent.Greenlet):
'''A 0MQ server for our communication with the user process'''
def __init__(self, input_list, output_list, ip_address=''):
def __init__(self, input_list, output_list, host_address):
super(Server, self).__init__()
......@@ -54,7 +67,8 @@ class Server(gevent.Greenlet):
# Starts our 0MQ server
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PAIR)
self.address = 'tcp://' + ip_address
self.address = 'tcp://' + get_network_ip_address(host_address)
port = self.socket.bind_to_random_port(self.address)
self.address += ':%d' % port
logger.debug("zmq server bound to `%s'", self.address)
......@@ -407,7 +421,8 @@ class Agent(object):
# Server for our single client
server = Server(configuration.input_list, configuration.output_list)
server = Server(configuration.input_list, configuration.output_list,
# Figures out the image to use
envkey = '%(name)s (%(version)s)' %['environment']
......@@ -428,7 +443,7 @@ class Agent(object):
cmd = ['sleep', str(daemon)]
logger.debug("Daemon mode: sleeping for %d seconds", daemon)
self.process = async.Popen(
self.process = dock.Popen(
......@@ -470,7 +485,7 @@ class Agent(object):
# Collects final information and returns to caller
process = self.process
self.process = None
return dict(
retval = dict(
stdout = process.stdout,
stderr = process.stderr,
status = status,
......@@ -480,6 +495,7 @@ class Agent(object):
user_error = server.user_error,
return retval
def kill(self):
......@@ -101,8 +101,7 @@ class Host(object):
if self.machine is not None:
return 'virtualbox@%s' % \
return self.kwargs['base_url']
return self.kwargs['base_url']
def env2docker(self, key):
......@@ -123,8 +122,6 @@ class Host(object):
self.started_machine = False
self.client = None
def __enter__(self):
......@@ -135,6 +132,13 @@ class Host(object):
def ip(self):
'''The IP address of the docker host'''
return '' if self.machine is None else self.machine.ip()
def _discover_environments(self, raise_on_errors=True):
'''Returns a dictionary containing information about docker environments
......@@ -156,7 +160,7 @@ class Host(object):
def _describe(image):
'''Tries to run the "describe" app on the image, collect results'''
status, output = self.get_statusoutput(image, 'describe')
status, output = self.get_statusoutput(image, ['describe'])
if status == 0:
return simplejson.loads(output)
......@@ -172,7 +176,7 @@ class Host(object):
tag = image['RepoTags'][0] if image['RepoTags'] else None
id = image['Id'].split(':')[1][:12]
logger.debug("Checking image `%s' (%s)...", tag, id)
description = _describe(image['Id'])
description = _describe(tag or id)
if not description: continue
key = description['name'] + ' (' + description['version'] + ')'
......@@ -185,7 +189,7 @@ class Host(object):
raise RuntimeError("Environments at `%s' and `%s' have the " \
"same name (`%s'). Distinct environments must be " \
"uniquely named. Fix this and re-start." % \
(envdir, retval[key]['image'], key))
(tag or id, retval[key]['image'], key))
logger.warn("Overriding **existing** environment `%s' image " \
"with `%s' (it was `%s). To avoid this warning make " \
......@@ -197,13 +201,60 @@ class Host(object):
retval[key]['image'] = image['Id']
retval[key]['tag'] = tag
retval[key]['short_id'] = id
retval[key]['nickname'] = tag or id"Registered `%s' -> `%s (%s)'", key, tag, id)
logger.debug("Found %d environments", len(retval))
return retval
def create_container(self, image, command, tmp_archive=None, **args):
def put_path(self, container, src, dest='/tmp', chmod=None):
"""Puts a given src path into a destination folder
This method will copy a given ``src`` path into a ``dest`` folder. It
optionally changes the owner of the resulting path on the image so it is
accessible by other users than root.
container (str, dict): The container where to copy the path.
src (str): The path, on the host, of the archive to be copied over the
container. The path may point to a file or directory on the host. Data
will be copied recursively.
dest (str, Optional): A path, on the container, where to copy the input
data. By default, we put data into the temporary directory of the
chmod (str, Optional): An optional string to set the mode of the
resulting path (files and directory). A common reason for this sort of
thing is to set directories to 755 but files to 644 with the following
value ``u+rwX,go+rX,go-w``. This will be applied recursively to the
resulting path on the container. If you set this value to ``None``,
then the chmod command will not be run.
# The docker API only accepts in-memory tar archives
c = six.moves.cStringIO()
with'w', fileobj=c) as tar:
tar.add(src, arcname=os.path.basename(src))
archive = c.getvalue()
# Place the tarball into the container
path = os.path.join(dest, os.path.basename(src))
logger.debug("[docker] archive -> %s@%s", container['Id'][:12], dest)
self.client.put_archive(container, dest, archive)
if chmod is not None:
# Change permissions to access the path
ex = self.client.exec_create(container, cmd=['chmod', '-R', chmod, dest])
output = self.client.exec_start(ex) #waits until it is executed
def create_container(self, image, command, tmp_path=None, **args):
"""Prepares the docker container for running the user code
......@@ -214,9 +265,9 @@ class Host(object):
command (list): A list of strings with the command to run inside the
tmp_archive (bytes): An archive to copy into the temporary directory of
the container (``/tmp``), supposedly with information that is used by
the command.
tmp_path (str): A path with a file name or directory that will be
copied into the container (inside ``/tmp``), supposedly with
information that is used by the command.
args (dict): A list of extra arguments to pass to the underlying
``create_container()`` call from the docker Python API.
......@@ -232,15 +283,27 @@ class Host(object):
if image in self: #replace by a real image name
image = self.env2docker(image)
# creates the log configuration, limiting output size kept on the image
config_args = dict(
log_config = docker.utils.LogConfig(type='',
config={'max-size': '1M', 'max-file': '1'}),
# if we use a virtual machine as docker host, then attach the container
# directly to the host as this will make it easier for our I/O server to
# connect to the contained application.
if self.use_machine:
config_args['network_mode'] = 'host'
args['host_config'] = self.client.create_host_config(**config_args)
logger.debug("[docker] create_container %s %s", image, ' '.join(command))
container = self.client.create_container(image=image, command=command,
if tmp_archive is not None:
# Place the tarball into the container
logger.debug("[docker] archive -> %s@/tmp", container['Id'][:12])
self.client.put_archive(container, '/tmp', tmp_archive)
if tmp_path is not None:
self.put_path(container, tmp_path)
return container
......@@ -296,27 +359,6 @@ class Host(object):
return status, output
def make_inmemory_tarball(path):
'''Creates an in-memory tarball of the contents of path
path (str): The path to a file or directory that will be tarred.
bytes: A byte-string representing the packaged object(s).
# Prepare the tarball, remove the temporary directory
c = six.moves.cStringIO()
with'w', fileobj=c) as tar:
tar.add(path, arcname=os.path.basename(path))
return c.getvalue()
class Popen:
'''Process-like manager for asynchronously executing user code in a container
......@@ -339,7 +381,7 @@ class Popen:
command (list): A list of strings with the command to run inside the
tmp_archive (str, bytes, Optional): An archive to copy into the temporary
path (str, Optional): An archive to copy into the temporary
directory of the container, supposedly with information that is used by
the command. If a string is given, than it is considered as a path from
which the archive is going to be created.
......@@ -361,27 +403,15 @@ class Popen: = host
'user': 'nobody',
'ports': ['5555'],
'user': 'root', #user `nobody' cannot access the tmp archive...
'tty': False,
'detach': True,
'stdin_open': False,
# creates the log configuration, limiting output size kept on the image
host_config =
log_config = docker.utils.LogConfig(type='',
config={'max-size': '1M', 'max-file': '1'}),
if isinstance(tmp_archive, six.string_types) and \
tmp_archive = make_inmemory_tarball(tmp_archive)
# creates the container
self.container =,
command=command, tmp_archive=tmp_archive, host_config=host_config,
command=command, tmp_path=tmp_archive, **args)
update_args = {}
......@@ -38,7 +38,7 @@ import pkg_resources
import docker
import requests
from ..async import Popen, Host
from ..dock import Popen, Host
# in case you want to see the printouts dynamically, set to ``True``
if False:
......@@ -53,14 +53,15 @@ if False:
class AsyncTest(unittest.TestCase):
def setUpClass(cls): = Host(use_machine='default')
def setUp(self): = Host(use_machine='default')
def tearDown(self):
def tearDownClass(cls):
def test_echo(self):
This diff is collapsed.
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment