diff --git a/bob/devtools/bootstrap.py b/bob/devtools/bootstrap.py index a0fda861340675ae987298f0e0cd4a43e5e36b06..e7d8b894d360298e652e5f0849029d540ba571e3 100644 --- a/bob/devtools/bootstrap.py +++ b/bob/devtools/bootstrap.py @@ -250,7 +250,7 @@ def install_miniconda(prefix, name): shutil.rmtree(cached) -def get_channels(public, stable, server, intranet): +def get_channels(public, stable, server, intranet, group): '''Returns the relevant conda channels to consider if building project The subset of channels to be returned depends on the visibility and stability @@ -274,6 +274,9 @@ def get_channels(public, stable, server, intranet): server: The base address of the server containing our conda channels intranet: Boolean indicating if we should add "private"/"public" prefixes on the conda paths + group: The group of packages (gitlab namespace) the package we're compiling + is part of. Values should match URL namespaces currently available on + our internal webserver. Currently, only "bob" or "beat" will work. Returns: a list of channels that need to be considered. @@ -287,12 +290,13 @@ def get_channels(public, stable, server, intranet): channels = [] if not public: - prefix = '/private' if intranet else '' + prefix = '/private' if not stable: #allowed private channels channels += [server + prefix + '/conda/label/beta'] #allowed betas channels += [server + prefix + '/conda'] - prefix = '/public' if intranet else '' + # do not use '/public' versions here + prefix = '/software/' + group if not stable: channels += [server + prefix + '/conda/label/beta'] #allowed betas channels += [server + prefix + '/conda'] @@ -416,7 +420,7 @@ if __name__ == '__main__': conda_bld_path = os.path.join(args.conda_root, 'conda-bld') run_cmdline([conda_bin, 'index', conda_bld_path]) channels = get_channels(public=True, stable=True, server=_SERVER, - intranet=True) + ['defaults'] + intranet=True, group='bob') + ['defaults'] channels = ['--override-channels'] + \ ['--channel=' + conda_bld_path] + \ ['--channel=%s' % k for k in channels] @@ -429,7 +433,7 @@ if __name__ == '__main__': # installs from channel channels = get_channels(public=True, stable=(args.tag is not None), - server=_SERVER, intranet=True) + ['defaults'] + server=_SERVER, intranet=True, group='bob') + ['defaults'] channels = ['--override-channels'] + ['--channel=%s' % k for k in channels] conda_cmd = 'install' if args.envname in ('base', 'root') else 'create' run_cmdline([conda_bin, conda_cmd] + conda_verbosity + channels + \ diff --git a/bob/devtools/build.py b/bob/devtools/build.py index 2f334efe98518766b65f9543cb8543e5f8a3954a..d82657ff5d2060aaaa7ae62d09c2b89922f81a21 100644 --- a/bob/devtools/build.py +++ b/bob/devtools/build.py @@ -322,7 +322,7 @@ def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local): yaml.dump(condarc, f, indent=2) -def get_docserver_setup(public, stable, server, intranet): +def get_docserver_setup(public, stable, server, intranet, group): '''Returns a setup for BOB_DOCUMENTATION_SERVER What is available to build the documentation depends on the setup of @@ -346,6 +346,9 @@ def get_docserver_setup(public, stable, server, intranet): server: The base address of the server containing our conda channels intranet: Boolean indicating if we should add "private"/"public" prefixes on the returned paths + group: The group of packages (gitlab namespace) the package we're compiling + is part of. Values should match URL namespaces currently available on + our internal webserver. Currently, only "bob" or "beat" will work. Returns: a string to be used by bob.extension to find dependent @@ -360,30 +363,28 @@ def get_docserver_setup(public, stable, server, intranet): entries = [] # public documentation: always can access - prefix = '/software/bob' - if server.endswith(prefix): # don't repeat yourself... - prefix = '' + prefix = '/software/%s' % group if stable: entries += [ - server + prefix + '/docs/bob/%(name)s/%(version)s/', - server + prefix + '/docs/bob/%(name)s/stable/', + server + prefix + '/docs/' + group + '/%(name)s/%(version)s/', + server + prefix + '/docs/' + group + '/%(name)s/stable/', ] else: entries += [ - server + prefix + '/docs/bob/%(name)s/master/', + server + prefix + '/docs/' + group + '/%(name)s/master/', ] if not public: # add private channels, (notice they are not accessible outside idiap) - prefix = '/private' if intranet else '' + prefix = '/private' if stable: entries += [ - server + prefix + '/docs/bob/%(name)s/%(version)s/', - server + prefix + '/docs/bob/%(name)s/stable/', + server + prefix + '/docs/' + group + '/%(name)s/%(version)s/', + server + prefix + '/docs/' + group + '/%(name)s/stable/', ] else: entries += [ - server + prefix + '/docs/bob/%(name)s/master/', + server + prefix + '/docs/' + group + '/%(name)s/master/', ] return '|'.join(entries) @@ -473,9 +474,9 @@ def git_clean_build(runner, verbose): ['--exclude=%s' % k for k in exclude_from_cleanup]) -def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config, - python_version, condarc_options): - '''Builds a non-beat/bob software dependence that does not exist on defaults +def base_build(bootstrap, server, intranet, group, recipe_dir, + conda_build_config, python_version, condarc_options): + '''Builds a non-beat/non-bob software dependence that doesn't exist on defaults This function will build a software dependence that is required for our software stack, but does not (yet) exist on the defaults channels. It first @@ -491,6 +492,9 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config, server: The base address of the server containing our conda channels intranet: Boolean indicating if we should add "private"/"public" prefixes on the returned paths + group: The group of packages (gitlab namespace) the package we're compiling + is part of. Values should match URL namespaces currently available on + our internal webserver. Currently, only "bob" or "beat" will work. recipe_dir: The directory containing the recipe's ``meta.yaml`` file conda_build_config: Path to the ``conda_build_config.yaml`` file to use python_version: String with the python version to build for, in the format @@ -505,7 +509,7 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config, # if you get to this point, tries to build the package public_channels = bootstrap.get_channels(public=True, stable=True, - server=server, intranet=intranet) + server=server, intranet=intranet, group=group) logger.info('Using the following channels during (potential) build:\n - %s', '\n - '.join(public_channels + ['defaults'])) @@ -566,6 +570,9 @@ if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Builds bob.devtools on the CI') + parser.add_argument('-g', '--group', + default=os.environ.get('CI_PROJECT_NAMESPACE', 'bob'), + help='The namespace of the project being built [default: %(default)s]') parser.add_argument('-n', '--name', default=os.environ.get('CI_PROJECT_NAME', 'bob.devtools'), help='The name of the project being built [default: %(default)s]') @@ -607,8 +614,7 @@ if __name__ == '__main__': spec = importlib.util.spec_from_file_location("bootstrap", bootstrap_file) bootstrap = importlib.util.module_from_spec(spec) spec.loader.exec_module(bootstrap) - server = bootstrap._SERVER if (not args.internet) else \ - 'https://www.idiap.ch/software/bob' + server = bootstrap._SERVER bootstrap.setup_logger(logger, args.verbose) @@ -640,7 +646,7 @@ if __name__ == '__main__': if not os.path.exists(os.path.join(recipe, 'meta.yaml')): # ignore - not a conda package continue - base_build(bootstrap, server, not args.internet, recipe, + base_build(bootstrap, server, not args.internet, group, recipe, conda_build_config, args.python_version, condarc_options) # notice this condarc typically will only contain the defaults channel - we @@ -648,7 +654,7 @@ if __name__ == '__main__': # build public = ( args.visibility == 'public' ) channels = bootstrap.get_channels(public=public, stable=(not is_prerelease), - server=server, intranet=(not args.internet)) + server=server, intranet=(not args.internet), group=group) logger.info('Using the following channels during build:\n - %s', '\n - '.join(channels + ['defaults'])) condarc_options['channels'] = channels + ['defaults'] diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py index 72924f7650391767a922c27db8d652d566e1d469..0674ebb9eb35c20b7ebd21bb809a946fafce0017 100644 --- a/bob/devtools/scripts/build.py +++ b/bob/devtools/scripts/build.py @@ -54,14 +54,16 @@ Examples: @click.option('-a', '--append-file', show_default=True, default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \ 'appended configuration file to use') -@click.option('-S', '--server', show_default=True, - default='https://www.idiap.ch/software/bob', help='Server used for ' \ - 'downloading conda packages and documentation indexes of required packages') +@click.option('-S', '--server', show_default=True, default=SERVER, + help='Server used for downloading conda packages and documentation ' \ + 'indexes of required packages') +@click.option('-g', '--group', show_default=True, default='bob', + help='Group of packages (gitlab namespace) this package belongs to') @click.option('-P', '--private/--no-private', default=False, help='Set this to **include** private channels on your build - ' \ 'you **must** be at Idiap to execute this build in this case - ' \ 'you **must** also use the correct server name through --server - ' \ - 'notice this option has no effect if you also pass --condarc') + 'notice this option has no effect to conda if you also pass --condarc') @click.option('-X', '--stable/--no-stable', default=False, help='Set this to **exclude** beta channels from your build - ' \ 'notice this option has no effect if you also pass --condarc') @@ -74,7 +76,7 @@ Examples: @verbosity_option() @bdt.raise_on_error def build(recipe_dir, python, condarc, config, no_test, append_file, - server, private, stable, dry_run, ci): + server, group, private, stable, dry_run, ci): """Builds package through conda-build with stock configuration This command wraps the execution of conda-build so that you use the same @@ -89,9 +91,12 @@ def build(recipe_dir, python, condarc, config, no_test, append_file, recipe_dir = recipe_dir or [os.path.join(os.path.realpath('.'), 'conda')] + logger.info('This package is considered part of group "%s" - tunning ' \ + 'conda package and documentation URLs for this...', group) + # get potential channel upload and other auxiliary channels channels = get_channels(public=(not private), stable=stable, server=server, - intranet=ci) + intranet=ci, group=group) if condarc is not None: logger.info('Loading CONDARC file from %s...', condarc) @@ -117,7 +122,7 @@ def build(recipe_dir, python, condarc, config, no_test, append_file, # and derived documentation building via Sphinx) set_environment('DOCSERVER', server) doc_urls = get_docserver_setup(public=(not private), stable=stable, - server=server, intranet=ci) + server=server, intranet=ci, group=group) set_environment('BOB_DOCUMENTATION_SERVER', doc_urls) for d in recipe_dir: diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py index 283e427c1569ed96e7182125e4f71f56d8d6a2c9..0a21c74106fb9022ed5e1e3d47b2732d07a3e5ab 100644 --- a/bob/devtools/scripts/ci.py +++ b/bob/devtools/scripts/ci.py @@ -317,6 +317,9 @@ Examples: ''') @click.argument('order', required=True, type=click.Path(file_okay=True, dir_okay=False, exists=True), nargs=1) +@click.option('-g', '--group', show_default=True, + default=os.environ['CI_PROJECT_NAMESPACE'], + help='Group of packages (gitlab namespace) this package belongs to') @click.option('-p', '--python', multiple=True, help='Versions of python in the format "x.y" we should build for. Pass ' \ 'various times this option to build for multiple python versions') @@ -326,7 +329,7 @@ Examples: 'printing to help you understand what will be done') @verbosity_option() @bdt.raise_on_error -def base_build(order, python, dry_run): +def base_build(order, group, python, dry_run): """Builds base (dependence) packages This command builds dependence packages (packages that are not Bob/BEAT @@ -335,7 +338,6 @@ def base_build(order, python, dry_run): """ from ..constants import CONDA_BUILD_CONFIG - from ..build import base_build as _build condarc = os.path.join(os.environ['CONDA_ROOT'], 'condarc') logger.info('Loading (this build\'s) CONDARC file from %s...', condarc) @@ -355,6 +357,7 @@ def base_build(order, python, dry_run): import itertools from .. import bootstrap + from ..build import base_build as _build # combine all versions of python with recipes if python: @@ -371,7 +374,7 @@ def base_build(order, python, dry_run): if not os.path.exists(os.path.join(recipe, 'meta.yaml')): logger.info('Ignoring directory "%s" - no meta.yaml found' % recipe) continue - _build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG, pyver, + _build(bootstrap, SERVER, True, group, recipe, CONDA_BUILD_CONFIG, pyver, condarc_options) @@ -399,6 +402,11 @@ def test(ctx, dry_run): from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND + group = os.environ['CI_PROJECT_NAMESPACE'] + if group not in ('bob', 'beat'): + # defaults back to bob - no other server setups are available as of now + group = 'bob' + from .test import test ctx.invoke(test, package = glob.glob(os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', @@ -407,6 +415,7 @@ def test(ctx, dry_run): config=CONDA_BUILD_CONFIG, append_file=CONDA_RECIPE_APPEND, server=SERVER, + group=group, private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'), stable='CI_COMMIT_TAG' in os.environ, dry_run=dry_run, @@ -438,6 +447,11 @@ def build(ctx, dry_run): from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND + group = os.environ['CI_PROJECT_NAMESPACE'] + if group not in ('bob', 'beat'): + # defaults back to bob - no other server setups are available as of now + group = 'bob' + from .build import build ctx.invoke(build, recipe_dir=[os.path.join(os.path.realpath(os.curdir), 'conda')], @@ -447,6 +461,7 @@ def build(ctx, dry_run): no_test=False, append_file=CONDA_RECIPE_APPEND, server=SERVER, + group=group, private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'), stable='CI_COMMIT_TAG' in os.environ, dry_run=dry_run, diff --git a/bob/devtools/scripts/create.py b/bob/devtools/scripts/create.py index 0d0b6b5efe1eb83b434106246ad65dcc875eec75..3783055583922694df43f93cf514a5f2e019f96e 100644 --- a/bob/devtools/scripts/create.py +++ b/bob/devtools/scripts/create.py @@ -72,9 +72,11 @@ Examples: @click.option('-a', '--append-file', show_default=True, default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \ 'appended configuration file to use') -@click.option('-S', '--server', show_default=True, - default='https://www.idiap.ch/software/bob', help='Server used for ' \ - 'downloading conda packages and documentation indexes of required packages') +@click.option('-S', '--server', show_default=True, default=SERVER, + help='Server used for downloading conda packages and documentation ' \ + 'indexes of required packages') +@click.option('-g', '--group', show_default=True, default='bob', + help='Group of packages (gitlab namespace) this package belongs to') @click.option('-P', '--private/--no-private', default=False, help='Set this to **include** private channels on your build - ' \ 'you **must** be at Idiap to execute this build in this case - ' \ @@ -123,6 +125,9 @@ def create(name, recipe_dir, python, overwrite, condarc, use_local, config, # set some environment variables before continuing set_environment('DOCSERVER', server, os.environ) + logger.info('This package is considered part of group "%s" - tunning ' \ + 'conda package URLs for this...', group) + if condarc is not None: logger.info('Loading CONDARC file from %s...', condarc) with open(condarc, 'rb') as f: @@ -131,7 +136,7 @@ def create(name, recipe_dir, python, overwrite, condarc, use_local, config, # use default and add channels condarc_options = yaml.load(BASE_CONDARC) #n.b.: no channels channels = get_channels(public=(not private), stable=stable, server=server, - intranet=private) + intranet=private, group=group) condarc_options['channels'] = channels + ['defaults'] conda_config = make_conda_config(config, python, append_file, condarc_options) diff --git a/bob/devtools/scripts/test.py b/bob/devtools/scripts/test.py index 605a4e363d8525d9b0ed806a891cf005afefc26c..7f7acd991dd122d565ed8c6c8001aa224d66b552 100644 --- a/bob/devtools/scripts/test.py +++ b/bob/devtools/scripts/test.py @@ -49,14 +49,16 @@ Examples: @click.option('-a', '--append-file', show_default=True, default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \ 'appended configuration file to use') -@click.option('-S', '--server', show_default=True, - default='https://www.idiap.ch/software/bob', help='Server used for ' \ - 'downloading conda packages and documentation indexes of required packages') +@click.option('-S', '--server', show_default=True, default=SERVER, + help='Server used for downloading conda packages and documentation ' \ + 'indexes of required packages') +@click.option('-g', '--group', show_default=True, default='bob', + help='Group of packages (gitlab namespace) this package belongs to') @click.option('-P', '--private/--no-private', default=False, help='Set this to **include** private channels on your build - ' \ 'you **must** be at Idiap to execute this build in this case - ' \ 'you **must** also use the correct server name through --server - ' \ - 'notice this option has no effect if you also pass --condarc') + 'notice this option has no effect to conda if you also pass --condarc') @click.option('-X', '--stable/--no-stable', default=False, help='Set this to **exclude** beta channels from your build - ' \ 'notice this option has no effect if you also pass --condarc') @@ -68,7 +70,7 @@ Examples: help='Use this flag to indicate the build will be running on the CI') @verbosity_option() @bdt.raise_on_error -def test(package, condarc, config, append_file, server, private, stable, +def test(package, condarc, config, append_file, server, group, private, stable, dry_run, ci): """Tests (pre-built) package through conda-build with stock configuration @@ -82,9 +84,12 @@ def test(package, condarc, config, append_file, server, private, stable, logger.warn('!!!! DRY RUN MODE !!!!') logger.warn('Nothing will be really built') + logger.info('This package is considered part of group "%s" - tunning ' \ + 'conda package and documentation URLs for this...', group) + # get potential channel upload and other auxiliary channels channels = get_channels(public=(not private), stable=stable, server=server, - intranet=ci) + intranet=ci, group=group) if condarc is not None: logger.info('Loading CONDARC file from %s...', condarc) @@ -110,7 +115,7 @@ def test(package, condarc, config, append_file, server, private, stable, # and derived documentation building via Sphinx) set_environment('DOCSERVER', server) doc_urls = get_docserver_setup(public=(not private), stable=stable, - server=server, intranet=ci) + server=server, intranet=ci, group=group) set_environment('BOB_DOCUMENTATION_SERVER', doc_urls) arch = conda_arch()