Merge branch 'package-namespace' into 'master'

Allows package namespace to be set and fine-tune conda/documentation urls

See merge request !24

bob/devtools/bootstrap.py

@@ -250,7 +250,7 @@ def install_miniconda(prefix, name):
     shutil.rmtree(cached)
 
 
-def get_channels(public, stable, server, intranet):
+def get_channels(public, stable, server, intranet, group):
   '''Returns the relevant conda channels to consider if building project
 
   The subset of channels to be returned depends on the visibility and stability
@@ -274,6 +274,9 @@ def get_channels(public, stable, server, intranet):
     server: The base address of the server containing our conda channels
     intranet: Boolean indicating if we should add "private"/"public" prefixes
       on the conda paths
+    group: The group of packages (gitlab namespace) the package we're compiling
+      is part of.  Values should match URL namespaces currently available on
+      our internal webserver.  Currently, only "bob" or "beat" will work.
 
 
   Returns: a list of channels that need to be considered.
@@ -287,12 +290,13 @@ def get_channels(public, stable, server, intranet):
   channels = []
 
   if not public:
-    prefix = '/private' if intranet else ''
+    prefix = '/private'
     if not stable:  #allowed private channels
       channels += [server + prefix + '/conda/label/beta']  #allowed betas
     channels += [server + prefix + '/conda']
 
-  prefix = '/public' if intranet else ''
+  # do not use '/public' versions here
+  prefix = '/software/' + group
   if not stable:
     channels += [server + prefix + '/conda/label/beta']  #allowed betas
   channels += [server + prefix + '/conda']
@@ -416,7 +420,7 @@ if __name__ == '__main__':
     conda_bld_path = os.path.join(args.conda_root, 'conda-bld')
     run_cmdline([conda_bin, 'index', conda_bld_path])
     channels = get_channels(public=True, stable=True, server=_SERVER,
-        intranet=True) + ['defaults']
+        intranet=True, group='bob') + ['defaults']
     channels = ['--override-channels'] + \
         ['--channel=' + conda_bld_path] + \
         ['--channel=%s' % k for k in channels]
@@ -429,7 +433,7 @@ if __name__ == '__main__':
     # installs from channel
     channels = get_channels(public=True,
         stable=(args.tag is not None),
-        server=_SERVER, intranet=True) + ['defaults']
+        server=_SERVER, intranet=True, group='bob') + ['defaults']
     channels = ['--override-channels'] + ['--channel=%s' % k for k in channels]
     conda_cmd = 'install' if args.envname in ('base', 'root') else 'create'
     run_cmdline([conda_bin, conda_cmd] + conda_verbosity + channels + \

bob/devtools/build.py

@@ -322,7 +322,7 @@ def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
       yaml.dump(condarc, f, indent=2)
 
 
-def get_docserver_setup(public, stable, server, intranet):
+def get_docserver_setup(public, stable, server, intranet, group):
   '''Returns a setup for BOB_DOCUMENTATION_SERVER
 
   What is available to build the documentation depends on the setup of
@@ -346,6 +346,9 @@ def get_docserver_setup(public, stable, server, intranet):
     server: The base address of the server containing our conda channels
     intranet: Boolean indicating if we should add "private"/"public" prefixes
       on the returned paths
+    group: The group of packages (gitlab namespace) the package we're compiling
+      is part of.  Values should match URL namespaces currently available on
+      our internal webserver.  Currently, only "bob" or "beat" will work.
 
 
   Returns: a string to be used by bob.extension to find dependent
@@ -360,30 +363,28 @@ def get_docserver_setup(public, stable, server, intranet):
   entries = []
 
   # public documentation: always can access
-  prefix = '/software/bob'
-  if server.endswith(prefix):  # don't repeat yourself...
-    prefix = ''
+  prefix = '/software/%s' % group
   if stable:
     entries += [
-        server + prefix + '/docs/bob/%(name)s/%(version)s/',
-        server + prefix + '/docs/bob/%(name)s/stable/',
+        server + prefix + '/docs/' + group + '/%(name)s/%(version)s/',
+        server + prefix + '/docs/' + group + '/%(name)s/stable/',
         ]
   else:
     entries += [
-        server + prefix + '/docs/bob/%(name)s/master/',
+        server + prefix + '/docs/' + group + '/%(name)s/master/',
         ]
 
   if not public:
     # add private channels, (notice they are not accessible outside idiap)
-    prefix = '/private' if intranet else ''
+    prefix = '/private'
     if stable:
       entries += [
-          server + prefix + '/docs/bob/%(name)s/%(version)s/',
-          server + prefix + '/docs/bob/%(name)s/stable/',
+          server + prefix + '/docs/' + group + '/%(name)s/%(version)s/',
+          server + prefix + '/docs/' + group + '/%(name)s/stable/',
           ]
     else:
       entries += [
-          server + prefix + '/docs/bob/%(name)s/master/',
+          server + prefix + '/docs/' + group + '/%(name)s/master/',
           ]
 
   return '|'.join(entries)
@@ -473,9 +474,9 @@ def git_clean_build(runner, verbose):
       ['--exclude=%s' % k for k in exclude_from_cleanup])
 
 
-def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
-    python_version, condarc_options):
-  '''Builds a non-beat/bob software dependence that does not exist on defaults
+def base_build(bootstrap, server, intranet, group, recipe_dir,
+    conda_build_config, python_version, condarc_options):
+  '''Builds a non-beat/non-bob software dependence that doesn't exist on defaults
 
   This function will build a software dependence that is required for our
   software stack, but does not (yet) exist on the defaults channels.  It first
@@ -491,6 +492,9 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
     server: The base address of the server containing our conda channels
     intranet: Boolean indicating if we should add "private"/"public" prefixes
       on the returned paths
+    group: The group of packages (gitlab namespace) the package we're compiling
+      is part of.  Values should match URL namespaces currently available on
+      our internal webserver.  Currently, only "bob" or "beat" will work.
     recipe_dir: The directory containing the recipe's ``meta.yaml`` file
     conda_build_config: Path to the ``conda_build_config.yaml`` file to use
     python_version: String with the python version to build for, in the format
@@ -505,7 +509,7 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
 
   # if you get to this point, tries to build the package
   public_channels = bootstrap.get_channels(public=True, stable=True,
-    server=server, intranet=intranet)
+    server=server, intranet=intranet, group=group)
 
   logger.info('Using the following channels during (potential) build:\n  - %s',
       '\n  - '.join(public_channels + ['defaults']))
@@ -566,6 +570,9 @@ if __name__ == '__main__':
   import argparse
 
   parser = argparse.ArgumentParser(description='Builds bob.devtools on the CI')
+  parser.add_argument('-g', '--group',
+      default=os.environ.get('CI_PROJECT_NAMESPACE', 'bob'),
+      help='The namespace of the project being built [default: %(default)s]')
   parser.add_argument('-n', '--name',
       default=os.environ.get('CI_PROJECT_NAME', 'bob.devtools'),
       help='The name of the project being built [default: %(default)s]')
@@ -607,8 +614,7 @@ if __name__ == '__main__':
   spec = importlib.util.spec_from_file_location("bootstrap", bootstrap_file)
   bootstrap = importlib.util.module_from_spec(spec)
   spec.loader.exec_module(bootstrap)
-  server = bootstrap._SERVER if (not args.internet) else \
-      'https://www.idiap.ch/software/bob'
+  server = bootstrap._SERVER
 
   bootstrap.setup_logger(logger, args.verbose)
 
@@ -640,7 +646,7 @@ if __name__ == '__main__':
     if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
       # ignore - not a conda package
       continue
-    base_build(bootstrap, server, not args.internet, recipe,
+    base_build(bootstrap, server, not args.internet, args.group, recipe,
         conda_build_config, args.python_version, condarc_options)
 
   # notice this condarc typically will only contain the defaults channel - we
@@ -648,7 +654,7 @@ if __name__ == '__main__':
   # build
   public = ( args.visibility == 'public' )
   channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
-      server=server, intranet=(not args.internet))
+      server=server, intranet=(not args.internet), group=args.group)
   logger.info('Using the following channels during build:\n  - %s',
       '\n  - '.join(channels + ['defaults']))
   condarc_options['channels'] = channels + ['defaults']

bob/devtools/scripts/build.py

@@ -54,14 +54,16 @@ Examples:
 @click.option('-a', '--append-file', show_default=True,
     default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
         'appended configuration file to use')
-@click.option('-S', '--server', show_default=True,
-    default='https://www.idiap.ch/software/bob', help='Server used for ' \
-    'downloading conda packages and documentation indexes of required packages')
+@click.option('-S', '--server', show_default=True, default=SERVER,
+    help='Server used for downloading conda packages and documentation ' \
+        'indexes of required packages')
+@click.option('-g', '--group', show_default=True, default='bob',
+    help='Group of packages (gitlab namespace) this package belongs to')
 @click.option('-P', '--private/--no-private', default=False,
     help='Set this to **include** private channels on your build - ' \
         'you **must** be at Idiap to execute this build in this case - ' \
         'you **must** also use the correct server name through --server - ' \
-        'notice this option has no effect if you also pass --condarc')
+        'notice this option has no effect to conda if you also pass --condarc')
 @click.option('-X', '--stable/--no-stable', default=False,
     help='Set this to **exclude** beta channels from your build - ' \
         'notice this option has no effect if you also pass --condarc')
@@ -74,7 +76,7 @@ Examples:
 @verbosity_option()
 @bdt.raise_on_error
 def build(recipe_dir, python, condarc, config, no_test, append_file,
-    server, private, stable, dry_run, ci):
+    server, group, private, stable, dry_run, ci):
   """Builds package through conda-build with stock configuration
 
   This command wraps the execution of conda-build so that you use the same
@@ -89,9 +91,12 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
 
   recipe_dir = recipe_dir or [os.path.join(os.path.realpath('.'), 'conda')]
 
+  logger.info('This package is considered part of group "%s" - tunning ' \
+      'conda package and documentation URLs for this...', group)
+
   # get potential channel upload and other auxiliary channels
   channels = get_channels(public=(not private), stable=stable, server=server,
-      intranet=ci)
+      intranet=ci, group=group)
 
   if condarc is not None:
     logger.info('Loading CONDARC file from %s...', condarc)
@@ -117,7 +122,7 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
   # and derived documentation building via Sphinx)
   set_environment('DOCSERVER', server)
   doc_urls = get_docserver_setup(public=(not private), stable=stable,
-      server=server, intranet=ci)
+      server=server, intranet=ci, group=group)
   set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
 
   for d in recipe_dir:

bob/devtools/scripts/ci.py

@@ -317,6 +317,9 @@ Examples:
 ''')
 @click.argument('order', required=True, type=click.Path(file_okay=True,
   dir_okay=False, exists=True), nargs=1)
+@click.option('-g', '--group', show_default=True,
+    default=os.environ['CI_PROJECT_NAMESPACE'],
+    help='Group of packages (gitlab namespace) this package belongs to')
 @click.option('-p', '--python', multiple=True,
     help='Versions of python in the format "x.y" we should build for.  Pass ' \
         'various times this option to build for multiple python versions')
@@ -326,7 +329,7 @@ Examples:
         'printing to help you understand what will be done')
 @verbosity_option()
 @bdt.raise_on_error
-def base_build(order, python, dry_run):
+def base_build(order, group, python, dry_run):
   """Builds base (dependence) packages
 
   This command builds dependence packages (packages that are not Bob/BEAT
@@ -335,7 +338,6 @@ def base_build(order, python, dry_run):
   """
 
   from ..constants import CONDA_BUILD_CONFIG
-  from ..build import base_build as _build
 
   condarc = os.path.join(os.environ['CONDA_ROOT'], 'condarc')
   logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
@@ -355,6 +357,7 @@ def base_build(order, python, dry_run):
 
   import itertools
   from .. import bootstrap
+  from ..build import base_build as _build
 
   # combine all versions of python with recipes
   if python:
@@ -371,7 +374,7 @@ def base_build(order, python, dry_run):
     if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
       logger.info('Ignoring directory "%s" - no meta.yaml found' % recipe)
       continue
-    _build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG, pyver,
+    _build(bootstrap, SERVER, True, group, recipe, CONDA_BUILD_CONFIG, pyver,
         condarc_options)
 
 
@@ -399,6 +402,11 @@ def test(ctx, dry_run):
 
   from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
 
+  group = os.environ['CI_PROJECT_NAMESPACE']
+  if group not in ('bob', 'beat'):
+    # defaults back to bob - no other server setups are available as of now
+    group = 'bob'
+
   from .test import test
   ctx.invoke(test,
       package = glob.glob(os.path.join(os.environ['CONDA_ROOT'], 'conda-bld',
@@ -407,6 +415,7 @@ def test(ctx, dry_run):
       config=CONDA_BUILD_CONFIG,
       append_file=CONDA_RECIPE_APPEND,
       server=SERVER,
+      group=group,
       private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'),
       stable='CI_COMMIT_TAG' in os.environ,
       dry_run=dry_run,
@@ -438,6 +447,11 @@ def build(ctx, dry_run):
 
   from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
 
+  group = os.environ['CI_PROJECT_NAMESPACE']
+  if group not in ('bob', 'beat'):
+    # defaults back to bob - no other server setups are available as of now
+    group = 'bob'
+
   from .build import build
   ctx.invoke(build,
       recipe_dir=[os.path.join(os.path.realpath(os.curdir), 'conda')],
@@ -447,6 +461,7 @@ def build(ctx, dry_run):
       no_test=False,
       append_file=CONDA_RECIPE_APPEND,
       server=SERVER,
+      group=group,
       private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'),
       stable='CI_COMMIT_TAG' in os.environ,
       dry_run=dry_run,

bob/devtools/scripts/create.py

@@ -72,9 +72,11 @@ Examples:
 @click.option('-a', '--append-file', show_default=True,
       default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
           'appended configuration file to use')
-@click.option('-S', '--server', show_default=True,
-    default='https://www.idiap.ch/software/bob', help='Server used for ' \
-    'downloading conda packages and documentation indexes of required packages')
+@click.option('-S', '--server', show_default=True, default=SERVER,
+    help='Server used for downloading conda packages and documentation ' \
+        'indexes of required packages')
+@click.option('-g', '--group', show_default=True, default='bob',
+    help='Group of packages (gitlab namespace) this package belongs to')
 @click.option('-P', '--private/--no-private', default=False,
     help='Set this to **include** private channels on your build - ' \
         'you **must** be at Idiap to execute this build in this case - ' \
@@ -123,6 +125,9 @@ def create(name, recipe_dir, python, overwrite, condarc, use_local, config,
   # set some environment variables before continuing
   set_environment('DOCSERVER', server, os.environ)
 
+  logger.info('This package is considered part of group "%s" - tunning ' \
+      'conda package URLs for this...', group)
+
   if condarc is not None:
     logger.info('Loading CONDARC file from %s...', condarc)
     with open(condarc, 'rb') as f:
@@ -131,7 +136,7 @@ def create(name, recipe_dir, python, overwrite, condarc, use_local, config,
     # use default and add channels
     condarc_options = yaml.load(BASE_CONDARC)  #n.b.: no channels
     channels = get_channels(public=(not private), stable=stable, server=server,
-        intranet=private)
+        intranet=private, group=group)
     condarc_options['channels'] = channels + ['defaults']
 
   conda_config = make_conda_config(config, python, append_file, condarc_options)

bob/devtools/scripts/test.py

@@ -49,14 +49,16 @@ Examples:
 @click.option('-a', '--append-file', show_default=True,
     default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
         'appended configuration file to use')
-@click.option('-S', '--server', show_default=True,
-    default='https://www.idiap.ch/software/bob', help='Server used for ' \
-    'downloading conda packages and documentation indexes of required packages')
+@click.option('-S', '--server', show_default=True, default=SERVER,
+    help='Server used for downloading conda packages and documentation ' \
+        'indexes of required packages')
+@click.option('-g', '--group', show_default=True, default='bob',
+    help='Group of packages (gitlab namespace) this package belongs to')
 @click.option('-P', '--private/--no-private', default=False,
     help='Set this to **include** private channels on your build - ' \
         'you **must** be at Idiap to execute this build in this case - ' \
         'you **must** also use the correct server name through --server - ' \
-        'notice this option has no effect if you also pass --condarc')
+        'notice this option has no effect to conda if you also pass --condarc')
 @click.option('-X', '--stable/--no-stable', default=False,
     help='Set this to **exclude** beta channels from your build - ' \
         'notice this option has no effect if you also pass --condarc')
@@ -68,7 +70,7 @@ Examples:
     help='Use this flag to indicate the build will be running on the CI')
 @verbosity_option()
 @bdt.raise_on_error
-def test(package, condarc, config, append_file, server, private, stable,
+def test(package, condarc, config, append_file, server, group, private, stable,
     dry_run, ci):
   """Tests (pre-built) package through conda-build with stock configuration
 
@@ -82,9 +84,12 @@ def test(package, condarc, config, append_file, server, private, stable,
       logger.warn('!!!! DRY RUN MODE !!!!')
       logger.warn('Nothing will be really built')
 
+  logger.info('This package is considered part of group "%s" - tunning ' \
+      'conda package and documentation URLs for this...', group)
+
   # get potential channel upload and other auxiliary channels
   channels = get_channels(public=(not private), stable=stable, server=server,
-      intranet=ci)
+      intranet=ci, group=group)
 
   if condarc is not None:
     logger.info('Loading CONDARC file from %s...', condarc)
@@ -110,7 +115,7 @@ def test(package, condarc, config, append_file, server, private, stable,
   # and derived documentation building via Sphinx)
   set_environment('DOCSERVER', server)
   doc_urls = get_docserver_setup(public=(not private), stable=stable,
-      server=server, intranet=ci)
+      server=server, intranet=ci, group=group)
   set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
 
   arch = conda_arch()