diff --git a/beat/web/backend/models.py b/beat/web/backend/models.py index eb6a447fb60e54b53710628840170720d1481d1b..4face51cc30dc5c26a72f25188a278d43007abb2 100755 --- a/beat/web/backend/models.py +++ b/beat/web/backend/models.py @@ -52,6 +52,7 @@ from guardian.shortcuts import get_perms import beat.core.stats import beat.core.data import beat.core.execution +from beat.core.dock import Host from ..code.models import Code from ..common.models import Shareable, ShareableManager @@ -474,31 +475,12 @@ class Worker(models.Model): status=Job.QUEUED, start_date__isnull=True, process_id__isnull=True) for split in splits: - execute = pick_execute(split, environments) - if execute is None: - message = "Environment `%s' is not available for split " \ - "%d/%d running at worker `%s', for block `%s' of " \ - "experiment `%s': %s" % \ - (split.job.block.environment, - split.split_index+1, - split.job.block.required_slots, - self, - split.job.block.name, - split.job.block.experiment.fullname(), - "Available environments are `%s'" % \ - '|'.join(environments.keys()), - ) - logger.error(message) - split.end(Result(status=1, - usrerr=settings.DEFAULT_USER_ERROR)) - continue - # if we get to this point, then we launch the user process # -> see settings.WORKER_DETACH_CHILDREN for more info kwargs = dict() if settings.WORKER_DETACH_CHILDREN: kwargs['preexec_fn'] = os.setpgrp - subprocess.Popen(cmdline + [execute, str(split.pk)], **kwargs) + subprocess.Popen(cmdline + ['execute', str(split.pk)], **kwargs) split.status = Job.PROCESSING #avoids re-running split.save() @@ -1140,6 +1122,9 @@ class JobSplit(models.Model): process_id = models.PositiveIntegerField(null=True) + host = None + + class Meta: unique_together = ('job', 'split_index') @@ -1446,6 +1431,10 @@ class JobSplit(models.Model): try: + if JobSplit.host is None: + JobSplit.host = Host() + JobSplit.host.setup() + executor = beat.core.execution.Executor(settings.PREFIX, config, cache) @@ -1475,10 +1464,9 @@ class JobSplit(models.Model): with executor: self.start() result = executor.process( - execute_path=execute, + JobSplit.host, virtual_memory_in_megabytes=queue.memory_limit, max_cpu_percent=int(100*float(nb_cores)), #allows for 150% - cpulimit_path=cpulimit, timeout_in_minutes=queue.time_limit, daemon=0, ) diff --git a/beat/web/backend/tests.py b/beat/web/backend/tests.py index f51d939576a6e1b5d3d936f7476a176b90a98584..c8b2c802eafdfdbe586d52f896b4f84bd6ea36d0 100755 --- a/beat/web/backend/tests.py +++ b/beat/web/backend/tests.py @@ -2099,7 +2099,7 @@ class Working(BaseBackendTestCase): self.cpulimit = utils.resolve_cpulimit_path(None) self.process = utils.resolve_process_path() self.environments = utils.find_environments(None) - self.env1_execute = self.environments['environment (1)']['execute'] + self.env1_execute = 'execute' if not os.path.exists(settings.CACHE_ROOT): os.makedirs(settings.CACHE_ROOT) @@ -2259,7 +2259,6 @@ class Working(BaseBackendTestCase): assert block.linear_execution_time() > 0.0 assert block.queuing_time() > 0.0 assert block.stdout() == '' - assert block.stderr() == '' assert block.error_report().find('Error') != -1 # assert we have no database traces after the block is done diff --git a/beat/web/backend/utils.py b/beat/web/backend/utils.py index 3ebfbe95909641e33df229fcef5a6cc3dd017046..8d0c0fa095bf2af58425bb3f3b61c5b6e9b45cfd 100755 --- a/beat/web/backend/utils.py +++ b/beat/web/backend/utils.py @@ -400,21 +400,11 @@ def find_environments(paths=None): ''' - from beat.core.execution import discover_environments - - if paths is not None: - logger.debug("Search for environments at `%s'", os.pathsep.join(paths)) - retval = discover_environments(paths) - logger.debug("Found %d environment(s)", len(retval)) - return retval - - else: - import pkg_resources - path = pkg_resources.resource_filename(__name__, 'environments') - logger.debug("Search for environments at `%s'", path) - retval = discover_environments([path]) - logger.debug("Found %d environment(s)", len(retval)) - return retval + from beat.core.dock import Host + + host = Host() + host.setup() + return host.environments def pick_execute(split, environments): diff --git a/beat/web/settings/test.py b/beat/web/settings/test.py index 9bb7d4dc121920241d630c0f8c4430553968aa88..178cbb721703b8a8274d92021e4d60fea0a385ed 100644 --- a/beat/web/settings/test.py +++ b/beat/web/settings/test.py @@ -46,6 +46,7 @@ if 'beat.cmdline' in sys.argv: LOGGING['handlers']['console']['level'] = 'DEBUG' LOGGING['loggers']['beat.core']['handlers'] = ['discard'] +LOGGING['loggers']['beat.web.utils.management.commands']['handlers'] = ['discard'] PREFIX = os.environ.get('BEAT_TEST_PREFIX', os.path.realpath('./test_prefix')) ALGORITHMS_ROOT = os.path.join(PREFIX, 'algorithms')