diff --git a/bob/devtools/scripts/jobs.py b/bob/devtools/scripts/jobs.py new file mode 100644 index 0000000000000000000000000000000000000000..4228520cd751e3fe8c3dc07beb4444719f950665 --- /dev/null +++ b/bob/devtools/scripts/jobs.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import os + +import click + +from . import bdt +from ..release import get_gitlab_instance + +from ..log import verbosity_option, get_logger, echo_normal +logger = get_logger(__name__) + + +@click.command(epilog=''' +Examples: + + 1. List running jobs on a runner defined by its description (macmini): + + $ bdt gitlab jobs -vv macmini + +''') +@click.argument('name') +@click.option('-s', '--status', type=click.Choice(['running', 'success', + 'failed', 'canceled']), + default='running', show_default=True, + help='The status of jobs we are searching for - one of "running", ' \ + '"success", "failed" or "canceled"') +@verbosity_option() +@bdt.raise_on_error +def jobs(name, status): + """Lists jobs on a given runner + """ + + gl = get_gitlab_instance() + gl.auth() + user_id = gl.user.attributes['id'] + + # search for the runner to affect + the_runner = [k for k in gl.runners.list(all=True) if \ + k.attributes['description'] == name] + if not the_runner: + raise RuntimeError('Cannot find runner with description = %s', name) + the_runner = the_runner[0] + logger.info('Found runner %s (id=%d)', + the_runner.attributes['description'], the_runner.attributes['id']) + + jobs = the_runner.jobs.list(all=True, status=status) + logger.info('There are %d jobs running on %s', len(jobs), name) + for k in jobs: + echo_normal('** job %d: %s (%s), since %s, by %s [%s]' % \ + (k.id, k.attributes['project']['path_with_namespace'], + k.attributes['name'], k.attributes['started_at'], + k.attributes['user']['username'], k.attributes['web_url'])) diff --git a/bob/devtools/scripts/runners.py b/bob/devtools/scripts/runners.py new file mode 100644 index 0000000000000000000000000000000000000000..d0a0c4a9676536fd9e7b338aa3439c58b07d597c --- /dev/null +++ b/bob/devtools/scripts/runners.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python + +import os + +import click + +from . import bdt +from ..release import get_gitlab_instance + +from ..log import verbosity_option, get_logger +logger = get_logger(__name__) + + +@click.command(epilog=''' +Examples: + + 1. Disables the runner with description "macmini" for all active projects in group "bob": + + $ bdt gitlab runners -vv bob disable macmini + + + 2. Enables the runner with description "linux-srv01" on all projects inside group "beat": + + $ bdt gitlab runners -vv beat enable linux-srv01 + + + 3. Enables the runner with description "linux-srv02" on a specific project: + + $ bdt gitlab runners -vv bob/bob.extension enable linux-srv02 + +''') +@click.argument('target') +@click.argument('cmd', type=click.Choice(['enable', 'disable'])) +@click.argument('name') +@click.option('-d', '--dry-run/--no-dry-run', default=False, + help='Only goes through the actions, but does not execute them ' \ + '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \ + 'printing to help you understand what will be done') +@verbosity_option() +@bdt.raise_on_error +def runners(target, cmd, name, dry_run): + """Enables and disables runners on whole gitlab groups or single projects + """ + + gl = get_gitlab_instance() + gl.auth() + user_id = gl.user.attributes['id'] + + if '/' in target: #it is a specific project + packages = [gl.projects.get(target)] + logger.debug('Found gitlab project %s (id=%d)', + packages[0].attributes['path_with_namespace'], packages[0].id) + + else: #it is a group - get all projects + logger.warn('Retrieving group by name - may take long...') + group = gl.groups.get(target) + logger.debug('Found gitlab group %s (id=%d)', group.attributes['path'], + group.id) + logger.warn('Retrieving all projects (with details) from group ' \ + '%s (id=%d)...', group.attributes['path'], group.id) + packages = [gl.projects.get(k.id) for k in \ + group.projects.list(all=True, simple=True)] + logger.info('Found %d projects under group %s', len(packages), + group.attributes['path']) + + # search for the runner to affect + the_runner = [k for k in gl.runners.list(all=True) if \ + k.attributes['description'] == name] + if not the_runner: + raise RuntimeError('Cannot find runner with description = %s', name) + the_runner = the_runner[0] + logger.info('Found runner %s (id=%d)', + the_runner.attributes['description'], the_runner.attributes['id']) + + for k in packages: + logger.info('Processing project %s (id=%d)', + k.attributes['path_with_namespace'], k.id) + + if cmd == 'enable': + + #checks if runner is not enabled first + enabled = False + for l in k.runners.list(all=True): + if l.id == the_runner.id: #it is there already + logger.warn('Runner %s (id=%d) is already enabled for project %s', + l.attributes['description'], l.id, + k.attributes['path_with_namespace']) + enabled = True + break + + if not enabled: #enable it + if not dry_run: + k.runners.create({'runner_id': the_runner.id}) + logger.info('Enabled runner %s (id=%d) for project %s', + the_runner.attributes['description'], the_runner.id, + k.attributes['path_with_namespace']) + + + elif cmd == 'disable': + + #checks if runner is not already disabled first + disabled = True + for l in k.runners.list(all=True): + if l.id == the_runner.id: #it is there already + logger.debug('Runner %s (id=%d) is enabled for project %s', + l.attributes['description'], l.id, + k.attributes['path_with_namespace']) + disabled = False + break + + if not disabled: #enable it + if not dry_run: + k.runners.delete(the_runner.id) + logger.info('Disabled runner %s (id=%d) for project %s', + the_runner.attributes['description'], the_runner.id, + k.attributes['path_with_namespace']) diff --git a/conda/meta.yaml b/conda/meta.yaml index 32af26cf0a6eacd964adc465e3667d6fcefd4b59..f5fdc7b3731619c9a812b8246d213fedfaa62f76 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -77,6 +77,8 @@ test: #- bdt gitlab visibility -vv bob/bob.devtools - bdt gitlab getpath --help #- bdt gitlab getpath -vv bob/bob.devtools .gitignore + - bdt gitlab runners --help + - bdt gitlab jobs --help - bdt ci --help - bdt ci base-build --help - bdt ci build --help diff --git a/doc/linux.rst b/doc/linux.rst index cba254333e82c29d2d9719dabfe398ea50007225..5ad04d8f6347cfbfc29b4e8b2319f14af4f1c9e3 100644 --- a/doc/linux.rst +++ b/doc/linux.rst @@ -166,3 +166,12 @@ The `docker-cleanup.sh` is: # Unused image leafs echo "Removing unused image leafs..." docker rmi $(docker images --filter "dangling=true" -q --no-trunc) + + +Conda and shared builds +======================= + +To avoid problems with conda and using shared builders, consider creating the +directory ``~gitlab-runner/.conda`` and touching the file +``environments.txt`` in that directory, setting a mode of ``444`` (i.e., make +it read-only). diff --git a/setup.py b/setup.py index ae8e42e99dbb11d0c3cbcda696cc5368a3ebb28b..2e852fa1440d33e3c4c3c188c450cb6c018bdb05 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,8 @@ setup( 'release = bob.devtools.scripts.release:release', 'changelog = bob.devtools.scripts.changelog:changelog', 'lasttag = bob.devtools.scripts.lasttag:lasttag', + 'runners = bob.devtools.scripts.runners:runners', + 'jobs = bob.devtools.scripts.jobs:jobs', 'visibility = bob.devtools.scripts.visibility:visibility', 'getpath = bob.devtools.scripts.getpath:getpath', ],