Commit 224ac477 authored by André Anjos's avatar André Anjos 💬

Remove docker-machine requirement and support code

parent 99652d6c
Pipeline #5743 failed with stage
in 6 minutes and 58 seconds
......@@ -47,24 +47,12 @@ class Host(object):
'''An object of this class can connect to the docker host and resolve stuff'''
def __init__(self, use_machine=None, **kwargs):
def __init__(self, **kwargs):
self.machine = None #normally, no need to use docker-machine
self.use_machine = None
self.started_machine = False
self.client = None
self.containers = []
if use_machine is not None:
import machine
self.machine = machine.Machine()
self.use_machine = use_machine
if not self.machine.exists(self.use_machine):
raise RuntimeError("no docker machine named `%s' found " \
" - perhaps you forgot to create it?" % self.use_machine)
elif 'host' in kwargs and 'port' in kwargs:
if 'host' in kwargs and 'port' in kwargs:
host = kwargs.get('host')
del kwargs['host']
port = kwargs.get('port')
......@@ -77,18 +65,7 @@ class Host(object):
def setup(self):
if self.use_machine is not None:
# requires a bootstrapping machine to run
if not self.machine.status(self.use_machine):
logger.debug("[docker-machine] start %s", self.use_machine)
self.machine.start(self.use_machine)
self.started_machine = True
self.client = \
docker.Client(**self.machine.config(machine=self.use_machine))
else:
self.client = docker.Client(**self.kwargs)
self.client = docker.Client(**self.kwargs)
self.environments = self._discover_environments(raise_on_errors=True)
......@@ -101,9 +78,6 @@ class Host(object):
def __str__(self):
if self.machine is not None:
return 'virtualbox@%s' % \
self.machine.config(machine=self.use_machine)['base_url']
return self.kwargs['base_url']
......@@ -120,11 +94,6 @@ class Host(object):
for container in self.containers: self.rm(container)
if self.use_machine is not None and self.started_machine:
logger.debug("[docker-machine] start %s", self.use_machine)
self.machine.stop(self.use_machine)
self.started_machine = False
def __enter__(self):
self.setup()
......@@ -139,7 +108,7 @@ class Host(object):
def ip(self):
'''The IP address of the docker host'''
return '127.0.0.1' if self.machine is None else self.machine.ip()
return '127.0.0.1'
def _discover_environments(self, raise_on_errors=True):
......@@ -257,7 +226,8 @@ class Host(object):
output = self.client.exec_start(ex) #waits until it is executed
def create_container(self, image, command, tmp_path=None, **args):
def create_container(self, image, command, tmp_path=None, host_args=None,
**args):
"""Prepares the docker container for running the user code
......@@ -272,6 +242,12 @@ class Host(object):
copied into the container (inside ``/tmp``), supposedly with
information that is used by the command.
host_args (dict): A dictionary that will be transformed into a
``HostConfig`` object that will be used for the creation of the
container. It is the place too add/drop capabilities for the container
or to set maximum resource usage (CPU, memory, etc). Keys accepted are
not arbitrary. Check docker_py's manual for details.
args (dict): A list of extra arguments to pass to the underlying
``create_container()`` call from the docker Python API.
......@@ -286,18 +262,21 @@ class Host(object):
if image in self: #replace by a real image name
image = self.env2docker(image)
# creates the log configuration, limiting output size kept on the image
config_args = dict(
log_config = docker.utils.LogConfig(type='',
config={'max-size': '1M', 'max-file': '1'}),
)
# if we use a virtual machine as docker host, then attach the container
# directly to the host as this will make it easier for our I/O server to
# connect to the contained application.
if self.use_machine:
config_args['network_mode'] = 'host'
if host_args is not None:
config_args.update(host_args)
# see bug: https://github.com/docker/docker-py/issues/1195
# fix on docker-engine 1.13.0 and superior
# see changelog: https://github.com/docker/docker/blob/master/CHANGELOG.md
args['network_disabled'] = False
# for HostConfig (keys are not arbitrary) - see:
# http://docker-py.readthedocs.io/en/latest/hostconfig/
args['host_config'] = self.client.create_host_config(**config_args)
logger.debug("[docker] create_container %s %s", image, ' '.join(command))
......@@ -375,9 +354,7 @@ class Popen:
host (:py:class:`Host`): The docker host where to start the container.
The host must be properly initialized, including starting the
appropriate docker-machine, if that is the case. This normally implies
including calls to this object inside a protected ``with host``
section.
appropriate docker-machine, if that is the case (environment setup).
image (str): A string uniquely identifying the image from which to start
the container
......@@ -413,22 +390,47 @@ class Popen:
'stdin_open': False,
})
# creates the container
self.container = self.host.create_container(image=image,
command=command, tmp_path=tmp_archive, **args)
host_args = dict()
update_args = {}
if virtual_memory_in_megabytes:
# For this to work properly, memory swap limitation has to be enabled on
# the kernel. This typically goes by setting "cgroup_enable=memory" as a
# boot parameter to kernels which are compiled with this support.
# More info: https://docs.docker.com/engine/installation/linux/ubuntulinux/#/enable-memory-and-swap-accounting
if max_cpu_percent:
update_args['cpu_period'] = 100000
update_args['cpu_quota'] = int(update_args['cpu_period'] * \
(max_cpu_percent/100.))
host_args['mem_limit'] = str(virtual_memory_in_megabytes) + 'm'
host_args['memswap_limit'] = host_args['mem_limit']
logger.debug('Setting maximum memory to %dMB' % \
(virtual_memory_in_megabytes,))
if virtual_memory_in_megabytes:
update_args['mem_limit'] = str(virtual_memory_in_megabytes) + 'm'
update_args['memswap_limit'] = update_args['mem_limit']
if max_cpu_percent:
# The period corresponds to the scheduling interval for the CFS in Linux
# The quota corresponds to a fraction or a multiple of the period, the
# container will get. A quota that is 2x the period gets the container up
# to 200% cpu time (2 cores). If the quota is 0.5x the period, the
# container gets up to 50% the cpu time. Each core represents 100%. A
# system with 2 cores has 200% computing power.
#
# More info:
# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
#
# For this to work properly, CPU bandwidth provisioning for the Linux CFS
# must be enabled on the kernel. More info on how to do it: http://www.blaess.fr/christophe/2012/01/07/linux-3-2-cfs-cpu-bandwidth-english-version/
#
# If your system is running on a virtual machine, having more cores
# available to docker engine normally translates to more precise
# scheduling.
period = 100000 #microseconds
quota = max_cpu_percent/100.
host_args['cpu_period'] = period
host_args['cpu_quota'] = int(period * quota)
logger.debug('Setting CPU quota to %d%%' % max_cpu_percent)
self.host.client.update_container(self.container, **update_args)
# creates the container
self.container = self.host.create_container(image=image,
command=command, tmp_path=tmp_archive,
host_args=host_args, **args)
# Starts the container
self.host.start(self.container)
......
......@@ -271,7 +271,8 @@ def cpu_statistics(start, end):
This method should be used when the currently set algorithm is the only one
executed through the whole process. It is done for collecting resource
statistics on separate processing environments.
statistics on separate processing environments. It follows the recipe in:
http://stackoverflow.com/questions/30271942/get-docker-container-cpu-usage-as-percentage
Returns:
......
......@@ -55,7 +55,7 @@ class AsyncTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.host = Host(use_machine='default')
cls.host = Host()
cls.host.setup()
......@@ -126,13 +126,13 @@ class AsyncTest(unittest.TestCase):
"print('Before')",
"import sys; sys.stdout.flush()",
"d = '0' * (10 * 1024 * 1024)",
"import time; time.sleep(3)",
"import time; time.sleep(5)",
"print('After')",
]),
], name='memory_limit', virtual_memory_in_megabytes=10) as p:
], name='memory_limit', virtual_memory_in_megabytes=4) as p:
#stats = p.statistics()
#print stats['memory']
time.sleep(2)
stats = p.statistics()
status = p.wait()
self.assertEqual(p.status(), 'exited')
self.assertEqual(status, 137)
......@@ -148,14 +148,18 @@ class AsyncTest(unittest.TestCase):
"print('Before')",
"import sys; sys.stdout.flush()",
"d = '0' * (10 * 1024 * 1024)",
"import time; time.sleep(3)",
"import time; time.sleep(5)",
"print('After')",
]),
], name='memory_limit2', virtual_memory_in_megabytes=100) as p:
time.sleep(2)
stats = p.statistics()
assert stats['memory']['percent'] < 15 and stats['memory']['percent'] > 10
status = p.wait()
assert stats['memory']['percent'] > 10, 'Memory check failed, ' \
'%d%% <= 10%%' % stats['memory']['percent']
assert stats['memory']['percent'] < 15, 'Memory check failed, ' \
'%d%% >= 15%%' % stats['memory']['percent']
self.assertEqual(p.status(), 'exited')
self.assertEqual(status, 0)
self.assertEqual(p.stdout.strip(), 'Before\nAfter')
......@@ -254,7 +258,7 @@ class AsyncTest(unittest.TestCase):
stats = p.statistics()
self.assertEqual(p.status(), 'running')
percent = stats['cpu']['percent']
assert percent < (1.05*max_cpu_percent), "%.2f%% is more than 5%% off the expected ceiling at %.2f%%!" % (percent, max_cpu_percent)
assert percent < (1.1*max_cpu_percent), "%.2f%% is more than 20%% off the expected ceiling at %d%%!" % (percent, max_cpu_percent)
# make sure nothing is there anymore
p.kill()
......@@ -264,8 +268,10 @@ class AsyncTest(unittest.TestCase):
def test_cpulimit_at_20percent(self):
# runs 1 process that should consume at most 20% of the host CPU
self._run_cpulimit(1, 20, 3)
def test_cpulimit_at_100percent(self):
# runs 4 processes that should consume 50% of the host CPU
self._run_cpulimit(4, 100, 3)
......@@ -59,7 +59,7 @@ class TestExecution(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.host = Host(use_machine='default')
cls.host = Host()
cls.host.setup()
......
......@@ -11,7 +11,7 @@ eggs = beat.core
coverage
[sources]
beat.backend.python = git git@gitlab.idiap.ch:beat/beat.backend.python
beat.backend.python = git https://gitlab.idiap.ch/beat/beat.backend.python
cpulimit = git https://github.com/opsengine/cpulimit rev=v0.2 egg=false
[cpulimit]
......@@ -29,7 +29,7 @@ recipe = bob.buildout:scripts
recipe = collective.recipe.cmd
cmds = cd beat/core/test/prefix/algorithms/user/
tar -cf cxx_integers_echo.tar cxx_integers_echo/
docker run -dti --name build beats/cxx_dev:0.1.2 > /dev/null
docker run -dti --name build beats/cxx_dev:0.1.5 > /dev/null
docker cp cxx_integers_echo.tar build:/tmp/cxx_integers_echo.tar
docker exec build bash -c 'cd /tmp ; tar -xf /tmp/cxx_integers_echo.tar'
docker exec build bash -c 'cd /tmp/cxx_integers_echo ; mkdir build ; cd build ; cmake .. ; make'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment