Commit a6eb8531 authored by André Anjos's avatar André Anjos 💬

Allows package namespace to be set and fine-tune conda/documentation urls

parent e19e1fed
Pipeline #27289 failed with stage
in 2 minutes and 6 seconds
......@@ -250,7 +250,7 @@ def install_miniconda(prefix, name):
shutil.rmtree(cached)
def get_channels(public, stable, server, intranet):
def get_channels(public, stable, server, intranet, group):
'''Returns the relevant conda channels to consider if building project
The subset of channels to be returned depends on the visibility and stability
......@@ -274,6 +274,9 @@ def get_channels(public, stable, server, intranet):
server: The base address of the server containing our conda channels
intranet: Boolean indicating if we should add "private"/"public" prefixes
on the conda paths
group: The group of packages (gitlab namespace) the package we're compiling
is part of. Values should match URL namespaces currently available on
our internal webserver. Currently, only "bob" or "beat" will work.
Returns: a list of channels that need to be considered.
......@@ -287,12 +290,13 @@ def get_channels(public, stable, server, intranet):
channels = []
if not public:
prefix = '/private' if intranet else ''
prefix = '/private'
if not stable: #allowed private channels
channels += [server + prefix + '/conda/label/beta'] #allowed betas
channels += [server + prefix + '/conda']
prefix = '/public' if intranet else ''
# do not use '/public' versions here
prefix = '/software/' + group
if not stable:
channels += [server + prefix + '/conda/label/beta'] #allowed betas
channels += [server + prefix + '/conda']
......@@ -416,7 +420,7 @@ if __name__ == '__main__':
conda_bld_path = os.path.join(args.conda_root, 'conda-bld')
run_cmdline([conda_bin, 'index', conda_bld_path])
channels = get_channels(public=True, stable=True, server=_SERVER,
intranet=True) + ['defaults']
intranet=True, group='bob') + ['defaults']
channels = ['--override-channels'] + \
['--channel=' + conda_bld_path] + \
['--channel=%s' % k for k in channels]
......@@ -429,7 +433,7 @@ if __name__ == '__main__':
# installs from channel
channels = get_channels(public=True,
stable=(args.tag is not None),
server=_SERVER, intranet=True) + ['defaults']
server=_SERVER, intranet=True, group='bob') + ['defaults']
channels = ['--override-channels'] + ['--channel=%s' % k for k in channels]
conda_cmd = 'install' if args.envname in ('base', 'root') else 'create'
run_cmdline([conda_bin, conda_cmd] + conda_verbosity + channels + \
......
......@@ -322,7 +322,7 @@ def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
yaml.dump(condarc, f, indent=2)
def get_docserver_setup(public, stable, server, intranet):
def get_docserver_setup(public, stable, server, intranet, group):
'''Returns a setup for BOB_DOCUMENTATION_SERVER
What is available to build the documentation depends on the setup of
......@@ -346,6 +346,9 @@ def get_docserver_setup(public, stable, server, intranet):
server: The base address of the server containing our conda channels
intranet: Boolean indicating if we should add "private"/"public" prefixes
on the returned paths
group: The group of packages (gitlab namespace) the package we're compiling
is part of. Values should match URL namespaces currently available on
our internal webserver. Currently, only "bob" or "beat" will work.
Returns: a string to be used by bob.extension to find dependent
......@@ -360,30 +363,28 @@ def get_docserver_setup(public, stable, server, intranet):
entries = []
# public documentation: always can access
prefix = '/software/bob'
if server.endswith(prefix): # don't repeat yourself...
prefix = ''
prefix = '/software/%s' % group
if stable:
entries += [
server + prefix + '/docs/bob/%(name)s/%(version)s/',
server + prefix + '/docs/bob/%(name)s/stable/',
server + prefix + '/docs/' + group + '/%(name)s/%(version)s/',
server + prefix + '/docs/' + group + '/%(name)s/stable/',
]
else:
entries += [
server + prefix + '/docs/bob/%(name)s/master/',
server + prefix + '/docs/' + group + '/%(name)s/master/',
]
if not public:
# add private channels, (notice they are not accessible outside idiap)
prefix = '/private' if intranet else ''
prefix = '/private'
if stable:
entries += [
server + prefix + '/docs/bob/%(name)s/%(version)s/',
server + prefix + '/docs/bob/%(name)s/stable/',
server + prefix + '/docs/' + group + '/%(name)s/%(version)s/',
server + prefix + '/docs/' + group + '/%(name)s/stable/',
]
else:
entries += [
server + prefix + '/docs/bob/%(name)s/master/',
server + prefix + '/docs/' + group + '/%(name)s/master/',
]
return '|'.join(entries)
......@@ -473,9 +474,9 @@ def git_clean_build(runner, verbose):
['--exclude=%s' % k for k in exclude_from_cleanup])
def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
python_version, condarc_options):
'''Builds a non-beat/bob software dependence that does not exist on defaults
def base_build(bootstrap, server, intranet, group, recipe_dir,
conda_build_config, python_version, condarc_options):
'''Builds a non-beat/non-bob software dependence that doesn't exist on defaults
This function will build a software dependence that is required for our
software stack, but does not (yet) exist on the defaults channels. It first
......@@ -491,6 +492,9 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
server: The base address of the server containing our conda channels
intranet: Boolean indicating if we should add "private"/"public" prefixes
on the returned paths
group: The group of packages (gitlab namespace) the package we're compiling
is part of. Values should match URL namespaces currently available on
our internal webserver. Currently, only "bob" or "beat" will work.
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use
python_version: String with the python version to build for, in the format
......@@ -505,7 +509,7 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
# if you get to this point, tries to build the package
public_channels = bootstrap.get_channels(public=True, stable=True,
server=server, intranet=intranet)
server=server, intranet=intranet, group=group)
logger.info('Using the following channels during (potential) build:\n - %s',
'\n - '.join(public_channels + ['defaults']))
......@@ -566,6 +570,9 @@ if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Builds bob.devtools on the CI')
parser.add_argument('-g', '--group',
default=os.environ.get('CI_PROJECT_NAMESPACE', 'bob'),
help='The namespace of the project being built [default: %(default)s]')
parser.add_argument('-n', '--name',
default=os.environ.get('CI_PROJECT_NAME', 'bob.devtools'),
help='The name of the project being built [default: %(default)s]')
......@@ -607,8 +614,7 @@ if __name__ == '__main__':
spec = importlib.util.spec_from_file_location("bootstrap", bootstrap_file)
bootstrap = importlib.util.module_from_spec(spec)
spec.loader.exec_module(bootstrap)
server = bootstrap._SERVER if (not args.internet) else \
'https://www.idiap.ch/software/bob'
server = bootstrap._SERVER
bootstrap.setup_logger(logger, args.verbose)
......@@ -640,7 +646,7 @@ if __name__ == '__main__':
if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
# ignore - not a conda package
continue
base_build(bootstrap, server, not args.internet, recipe,
base_build(bootstrap, server, not args.internet, group, recipe,
conda_build_config, args.python_version, condarc_options)
# notice this condarc typically will only contain the defaults channel - we
......@@ -648,7 +654,7 @@ if __name__ == '__main__':
# build
public = ( args.visibility == 'public' )
channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
server=server, intranet=(not args.internet))
server=server, intranet=(not args.internet), group=group)
logger.info('Using the following channels during build:\n - %s',
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
......
......@@ -54,14 +54,16 @@ Examples:
@click.option('-a', '--append-file', show_default=True,
default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
'appended configuration file to use')
@click.option('-S', '--server', show_default=True,
default='https://www.idiap.ch/software/bob', help='Server used for ' \
'downloading conda packages and documentation indexes of required packages')
@click.option('-S', '--server', show_default=True, default=SERVER,
help='Server used for downloading conda packages and documentation ' \
'indexes of required packages')
@click.option('-g', '--group', show_default=True, default='bob',
help='Group of packages (gitlab namespace) this package belongs to')
@click.option('-P', '--private/--no-private', default=False,
help='Set this to **include** private channels on your build - ' \
'you **must** be at Idiap to execute this build in this case - ' \
'you **must** also use the correct server name through --server - ' \
'notice this option has no effect if you also pass --condarc')
'notice this option has no effect to conda if you also pass --condarc')
@click.option('-X', '--stable/--no-stable', default=False,
help='Set this to **exclude** beta channels from your build - ' \
'notice this option has no effect if you also pass --condarc')
......@@ -74,7 +76,7 @@ Examples:
@verbosity_option()
@bdt.raise_on_error
def build(recipe_dir, python, condarc, config, no_test, append_file,
server, private, stable, dry_run, ci):
server, group, private, stable, dry_run, ci):
"""Builds package through conda-build with stock configuration
This command wraps the execution of conda-build so that you use the same
......@@ -89,9 +91,12 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
recipe_dir = recipe_dir or [os.path.join(os.path.realpath('.'), 'conda')]
logger.info('This package is considered part of group "%s" - tunning ' \
'conda package and documentation URLs for this...', group)
# get potential channel upload and other auxiliary channels
channels = get_channels(public=(not private), stable=stable, server=server,
intranet=ci)
intranet=ci, group=group)
if condarc is not None:
logger.info('Loading CONDARC file from %s...', condarc)
......@@ -117,7 +122,7 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
# and derived documentation building via Sphinx)
set_environment('DOCSERVER', server)
doc_urls = get_docserver_setup(public=(not private), stable=stable,
server=server, intranet=ci)
server=server, intranet=ci, group=group)
set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
for d in recipe_dir:
......
......@@ -317,6 +317,9 @@ Examples:
''')
@click.argument('order', required=True, type=click.Path(file_okay=True,
dir_okay=False, exists=True), nargs=1)
@click.option('-g', '--group', show_default=True,
default=os.environ['CI_PROJECT_NAMESPACE'],
help='Group of packages (gitlab namespace) this package belongs to')
@click.option('-p', '--python', multiple=True,
help='Versions of python in the format "x.y" we should build for. Pass ' \
'various times this option to build for multiple python versions')
......@@ -326,7 +329,7 @@ Examples:
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def base_build(order, python, dry_run):
def base_build(order, group, python, dry_run):
"""Builds base (dependence) packages
This command builds dependence packages (packages that are not Bob/BEAT
......@@ -335,7 +338,6 @@ def base_build(order, python, dry_run):
"""
from ..constants import CONDA_BUILD_CONFIG
from ..build import base_build as _build
condarc = os.path.join(os.environ['CONDA_ROOT'], 'condarc')
logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
......@@ -355,6 +357,7 @@ def base_build(order, python, dry_run):
import itertools
from .. import bootstrap
from ..build import base_build as _build
# combine all versions of python with recipes
if python:
......@@ -371,7 +374,7 @@ def base_build(order, python, dry_run):
if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
logger.info('Ignoring directory "%s" - no meta.yaml found' % recipe)
continue
_build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG, pyver,
_build(bootstrap, SERVER, True, group, recipe, CONDA_BUILD_CONFIG, pyver,
condarc_options)
......@@ -399,6 +402,11 @@ def test(ctx, dry_run):
from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
group = os.environ['CI_PROJECT_NAMESPACE']
if group not in ('bob', 'beat'):
# defaults back to bob - no other server setups are available as of now
group = 'bob'
from .test import test
ctx.invoke(test,
package = glob.glob(os.path.join(os.environ['CONDA_ROOT'], 'conda-bld',
......@@ -407,6 +415,7 @@ def test(ctx, dry_run):
config=CONDA_BUILD_CONFIG,
append_file=CONDA_RECIPE_APPEND,
server=SERVER,
group=group,
private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'),
stable='CI_COMMIT_TAG' in os.environ,
dry_run=dry_run,
......@@ -438,6 +447,11 @@ def build(ctx, dry_run):
from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
group = os.environ['CI_PROJECT_NAMESPACE']
if group not in ('bob', 'beat'):
# defaults back to bob - no other server setups are available as of now
group = 'bob'
from .build import build
ctx.invoke(build,
recipe_dir=[os.path.join(os.path.realpath(os.curdir), 'conda')],
......@@ -447,6 +461,7 @@ def build(ctx, dry_run):
no_test=False,
append_file=CONDA_RECIPE_APPEND,
server=SERVER,
group=group,
private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'),
stable='CI_COMMIT_TAG' in os.environ,
dry_run=dry_run,
......
......@@ -72,9 +72,11 @@ Examples:
@click.option('-a', '--append-file', show_default=True,
default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
'appended configuration file to use')
@click.option('-S', '--server', show_default=True,
default='https://www.idiap.ch/software/bob', help='Server used for ' \
'downloading conda packages and documentation indexes of required packages')
@click.option('-S', '--server', show_default=True, default=SERVER,
help='Server used for downloading conda packages and documentation ' \
'indexes of required packages')
@click.option('-g', '--group', show_default=True, default='bob',
help='Group of packages (gitlab namespace) this package belongs to')
@click.option('-P', '--private/--no-private', default=False,
help='Set this to **include** private channels on your build - ' \
'you **must** be at Idiap to execute this build in this case - ' \
......@@ -123,6 +125,9 @@ def create(name, recipe_dir, python, overwrite, condarc, use_local, config,
# set some environment variables before continuing
set_environment('DOCSERVER', server, os.environ)
logger.info('This package is considered part of group "%s" - tunning ' \
'conda package URLs for this...', group)
if condarc is not None:
logger.info('Loading CONDARC file from %s...', condarc)
with open(condarc, 'rb') as f:
......@@ -131,7 +136,7 @@ def create(name, recipe_dir, python, overwrite, condarc, use_local, config,
# use default and add channels
condarc_options = yaml.load(BASE_CONDARC) #n.b.: no channels
channels = get_channels(public=(not private), stable=stable, server=server,
intranet=private)
intranet=private, group=group)
condarc_options['channels'] = channels + ['defaults']
conda_config = make_conda_config(config, python, append_file, condarc_options)
......
......@@ -49,14 +49,16 @@ Examples:
@click.option('-a', '--append-file', show_default=True,
default=CONDA_RECIPE_APPEND, help='overwrites the path leading to ' \
'appended configuration file to use')
@click.option('-S', '--server', show_default=True,
default='https://www.idiap.ch/software/bob', help='Server used for ' \
'downloading conda packages and documentation indexes of required packages')
@click.option('-S', '--server', show_default=True, default=SERVER,
help='Server used for downloading conda packages and documentation ' \
'indexes of required packages')
@click.option('-g', '--group', show_default=True, default='bob',
help='Group of packages (gitlab namespace) this package belongs to')
@click.option('-P', '--private/--no-private', default=False,
help='Set this to **include** private channels on your build - ' \
'you **must** be at Idiap to execute this build in this case - ' \
'you **must** also use the correct server name through --server - ' \
'notice this option has no effect if you also pass --condarc')
'notice this option has no effect to conda if you also pass --condarc')
@click.option('-X', '--stable/--no-stable', default=False,
help='Set this to **exclude** beta channels from your build - ' \
'notice this option has no effect if you also pass --condarc')
......@@ -68,7 +70,7 @@ Examples:
help='Use this flag to indicate the build will be running on the CI')
@verbosity_option()
@bdt.raise_on_error
def test(package, condarc, config, append_file, server, private, stable,
def test(package, condarc, config, append_file, server, group, private, stable,
dry_run, ci):
"""Tests (pre-built) package through conda-build with stock configuration
......@@ -82,9 +84,12 @@ def test(package, condarc, config, append_file, server, private, stable,
logger.warn('!!!! DRY RUN MODE !!!!')
logger.warn('Nothing will be really built')
logger.info('This package is considered part of group "%s" - tunning ' \
'conda package and documentation URLs for this...', group)
# get potential channel upload and other auxiliary channels
channels = get_channels(public=(not private), stable=stable, server=server,
intranet=ci)
intranet=ci, group=group)
if condarc is not None:
logger.info('Loading CONDARC file from %s...', condarc)
......@@ -110,7 +115,7 @@ def test(package, condarc, config, append_file, server, private, stable,
# and derived documentation building via Sphinx)
set_environment('DOCSERVER', server)
doc_urls = get_docserver_setup(public=(not private), stable=stable,
server=server, intranet=ci)
server=server, intranet=ci, group=group)
set_environment('BOB_DOCUMENTATION_SERVER', doc_urls)
arch = conda_arch()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment