Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.devtools
Commits
a6eb8531
Commit
a6eb8531
authored
Feb 20, 2019
by
André Anjos
💬
Browse files
Allows package namespace to be set and fine-tune conda/documentation urls
parent
e19e1fed
Pipeline
#27289
failed with stage
in 2 minutes and 6 seconds
Changes
6
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
bob/devtools/bootstrap.py
View file @
a6eb8531
...
...
@@ -250,7 +250,7 @@ def install_miniconda(prefix, name):
shutil
.
rmtree
(
cached
)
def
get_channels
(
public
,
stable
,
server
,
intranet
):
def
get_channels
(
public
,
stable
,
server
,
intranet
,
group
):
'''Returns the relevant conda channels to consider if building project
The subset of channels to be returned depends on the visibility and stability
...
...
@@ -274,6 +274,9 @@ def get_channels(public, stable, server, intranet):
server: The base address of the server containing our conda channels
intranet: Boolean indicating if we should add "private"/"public" prefixes
on the conda paths
group: The group of packages (gitlab namespace) the package we're compiling
is part of. Values should match URL namespaces currently available on
our internal webserver. Currently, only "bob" or "beat" will work.
Returns: a list of channels that need to be considered.
...
...
@@ -287,12 +290,13 @@ def get_channels(public, stable, server, intranet):
channels
=
[]
if
not
public
:
prefix
=
'/private'
if
intranet
else
''
prefix
=
'/private'
if
not
stable
:
#allowed private channels
channels
+=
[
server
+
prefix
+
'/conda/label/beta'
]
#allowed betas
channels
+=
[
server
+
prefix
+
'/conda'
]
prefix
=
'/public'
if
intranet
else
''
# do not use '/public' versions here
prefix
=
'/software/'
+
group
if
not
stable
:
channels
+=
[
server
+
prefix
+
'/conda/label/beta'
]
#allowed betas
channels
+=
[
server
+
prefix
+
'/conda'
]
...
...
@@ -416,7 +420,7 @@ if __name__ == '__main__':
conda_bld_path
=
os
.
path
.
join
(
args
.
conda_root
,
'conda-bld'
)
run_cmdline
([
conda_bin
,
'index'
,
conda_bld_path
])
channels
=
get_channels
(
public
=
True
,
stable
=
True
,
server
=
_SERVER
,
intranet
=
True
)
+
[
'defaults'
]
intranet
=
True
,
group
=
'bob'
)
+
[
'defaults'
]
channels
=
[
'--override-channels'
]
+
\
[
'--channel='
+
conda_bld_path
]
+
\
[
'--channel=%s'
%
k
for
k
in
channels
]
...
...
@@ -429,7 +433,7 @@ if __name__ == '__main__':
# installs from channel
channels
=
get_channels
(
public
=
True
,
stable
=
(
args
.
tag
is
not
None
),
server
=
_SERVER
,
intranet
=
True
)
+
[
'defaults'
]
server
=
_SERVER
,
intranet
=
True
,
group
=
'bob'
)
+
[
'defaults'
]
channels
=
[
'--override-channels'
]
+
[
'--channel=%s'
%
k
for
k
in
channels
]
conda_cmd
=
'install'
if
args
.
envname
in
(
'base'
,
'root'
)
else
'create'
run_cmdline
([
conda_bin
,
conda_cmd
]
+
conda_verbosity
+
channels
+
\
...
...
bob/devtools/build.py
View file @
a6eb8531
...
...
@@ -322,7 +322,7 @@ def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
yaml
.
dump
(
condarc
,
f
,
indent
=
2
)
def
get_docserver_setup
(
public
,
stable
,
server
,
intranet
):
def
get_docserver_setup
(
public
,
stable
,
server
,
intranet
,
group
):
'''Returns a setup for BOB_DOCUMENTATION_SERVER
What is available to build the documentation depends on the setup of
...
...
@@ -346,6 +346,9 @@ def get_docserver_setup(public, stable, server, intranet):
server: The base address of the server containing our conda channels
intranet: Boolean indicating if we should add "private"/"public" prefixes
on the returned paths
group: The group of packages (gitlab namespace) the package we're compiling
is part of. Values should match URL namespaces currently available on
our internal webserver. Currently, only "bob" or "beat" will work.
Returns: a string to be used by bob.extension to find dependent
...
...
@@ -360,30 +363,28 @@ def get_docserver_setup(public, stable, server, intranet):
entries
=
[]
# public documentation: always can access
prefix
=
'/software/bob'
if
server
.
endswith
(
prefix
):
# don't repeat yourself...
prefix
=
''
prefix
=
'/software/%s'
%
group
if
stable
:
entries
+=
[
server
+
prefix
+
'/docs/
bob
/%(name)s/%(version)s/'
,
server
+
prefix
+
'/docs/
bob
/%(name)s/stable/'
,
server
+
prefix
+
'/docs/
'
+
group
+
'
/%(name)s/%(version)s/'
,
server
+
prefix
+
'/docs/
'
+
group
+
'
/%(name)s/stable/'
,
]
else
:
entries
+=
[
server
+
prefix
+
'/docs/
bob
/%(name)s/master/'
,
server
+
prefix
+
'/docs/
'
+
group
+
'
/%(name)s/master/'
,
]
if
not
public
:
# add private channels, (notice they are not accessible outside idiap)
prefix
=
'/private'
if
intranet
else
''
prefix
=
'/private'
if
stable
:
entries
+=
[
server
+
prefix
+
'/docs/
bob
/%(name)s/%(version)s/'
,
server
+
prefix
+
'/docs/
bob
/%(name)s/stable/'
,
server
+
prefix
+
'/docs/
'
+
group
+
'
/%(name)s/%(version)s/'
,
server
+
prefix
+
'/docs/
'
+
group
+
'
/%(name)s/stable/'
,
]
else
:
entries
+=
[
server
+
prefix
+
'/docs/
bob
/%(name)s/master/'
,
server
+
prefix
+
'/docs/
'
+
group
+
'
/%(name)s/master/'
,
]
return
'|'
.
join
(
entries
)
...
...
@@ -473,9 +474,9 @@ def git_clean_build(runner, verbose):
[
'--exclude=%s'
%
k
for
k
in
exclude_from_cleanup
])
def
base_build
(
bootstrap
,
server
,
intranet
,
recipe_dir
,
conda_build_config
,
python_version
,
condarc_options
):
'''Builds a non-beat/bob software dependence that does
no
t exist on defaults
def
base_build
(
bootstrap
,
server
,
intranet
,
group
,
recipe_dir
,
conda_build_config
,
python_version
,
condarc_options
):
'''Builds a non-beat/
non-
bob software dependence that does
n'
t exist on defaults
This function will build a software dependence that is required for our
software stack, but does not (yet) exist on the defaults channels. It first
...
...
@@ -491,6 +492,9 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
server: The base address of the server containing our conda channels
intranet: Boolean indicating if we should add "private"/"public" prefixes
on the returned paths
group: The group of packages (gitlab namespace) the package we're compiling
is part of. Values should match URL namespaces currently available on
our internal webserver. Currently, only "bob" or "beat" will work.
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use
python_version: String with the python version to build for, in the format
...
...
@@ -505,7 +509,7 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
# if you get to this point, tries to build the package
public_channels
=
bootstrap
.
get_channels
(
public
=
True
,
stable
=
True
,
server
=
server
,
intranet
=
intranet
)
server
=
server
,
intranet
=
intranet
,
group
=
group
)
logger
.
info
(
'Using the following channels during (potential) build:
\n
- %s'
,
'
\n
- '
.
join
(
public_channels
+
[
'defaults'
]))
...
...
@@ -566,6 +570,9 @@ if __name__ == '__main__':
import
argparse
parser
=
argparse
.
ArgumentParser
(
description
=
'Builds bob.devtools on the CI'
)
parser
.
add_argument
(
'-g'
,
'--group'
,
default
=
os
.
environ
.
get
(
'CI_PROJECT_NAMESPACE'
,
'bob'
),
help
=
'The namespace of the project being built [default: %(default)s]'
)
parser
.
add_argument
(
'-n'
,
'--name'
,
default
=
os
.
environ
.
get
(
'CI_PROJECT_NAME'
,
'bob.devtools'
),
help
=
'The name of the project being built [default: %(default)s]'
)
...
...
@@ -607,8 +614,7 @@ if __name__ == '__main__':
spec
=
importlib
.
util
.
spec_from_file_location
(
"bootstrap"
,
bootstrap_file
)
bootstrap
=
importlib
.
util
.
module_from_spec
(
spec
)
spec
.
loader
.
exec_module
(
bootstrap
)
server
=
bootstrap
.
_SERVER
if
(
not
args
.
internet
)
else
\
'https://www.idiap.ch/software/bob'
server
=
bootstrap
.
_SERVER
bootstrap
.
setup_logger
(
logger
,
args
.
verbose
)
...
...
@@ -640,7 +646,7 @@ if __name__ == '__main__':
if
not
os
.
path
.
exists
(
os
.
path
.
join
(
recipe
,
'meta.yaml'
)):
# ignore - not a conda package
continue
base_build
(
bootstrap
,
server
,
not
args
.
internet
,
recipe
,
base_build
(
bootstrap
,
server
,
not
args
.
internet
,
group
,
recipe
,
conda_build_config
,
args
.
python_version
,
condarc_options
)
# notice this condarc typically will only contain the defaults channel - we
...
...
@@ -648,7 +654,7 @@ if __name__ == '__main__':
# build
public
=
(
args
.
visibility
==
'public'
)
channels
=
bootstrap
.
get_channels
(
public
=
public
,
stable
=
(
not
is_prerelease
),
server
=
server
,
intranet
=
(
not
args
.
internet
))
server
=
server
,
intranet
=
(
not
args
.
internet
)
,
group
=
group
)
logger
.
info
(
'Using the following channels during build:
\n
- %s'
,
'
\n
- '
.
join
(
channels
+
[
'defaults'
]))
condarc_options
[
'channels'
]
=
channels
+
[
'defaults'
]
...
...
bob/devtools/scripts/build.py
View file @
a6eb8531
...
...
@@ -54,14 +54,16 @@ Examples:
@
click
.
option
(
'-a'
,
'--append-file'
,
show_default
=
True
,
default
=
CONDA_RECIPE_APPEND
,
help
=
'overwrites the path leading to '
\
'appended configuration file to use'
)
@
click
.
option
(
'-S'
,
'--server'
,
show_default
=
True
,
default
=
'https://www.idiap.ch/software/bob'
,
help
=
'Server used for '
\
'downloading conda packages and documentation indexes of required packages'
)
@
click
.
option
(
'-S'
,
'--server'
,
show_default
=
True
,
default
=
SERVER
,
help
=
'Server used for downloading conda packages and documentation '
\
'indexes of required packages'
)
@
click
.
option
(
'-g'
,
'--group'
,
show_default
=
True
,
default
=
'bob'
,
help
=
'Group of packages (gitlab namespace) this package belongs to'
)
@
click
.
option
(
'-P'
,
'--private/--no-private'
,
default
=
False
,
help
=
'Set this to **include** private channels on your build - '
\
'you **must** be at Idiap to execute this build in this case - '
\
'you **must** also use the correct server name through --server - '
\
'notice this option has no effect if you also pass --condarc'
)
'notice this option has no effect
to conda
if you also pass --condarc'
)
@
click
.
option
(
'-X'
,
'--stable/--no-stable'
,
default
=
False
,
help
=
'Set this to **exclude** beta channels from your build - '
\
'notice this option has no effect if you also pass --condarc'
)
...
...
@@ -74,7 +76,7 @@ Examples:
@
verbosity_option
()
@
bdt
.
raise_on_error
def
build
(
recipe_dir
,
python
,
condarc
,
config
,
no_test
,
append_file
,
server
,
private
,
stable
,
dry_run
,
ci
):
server
,
group
,
private
,
stable
,
dry_run
,
ci
):
"""Builds package through conda-build with stock configuration
This command wraps the execution of conda-build so that you use the same
...
...
@@ -89,9 +91,12 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
recipe_dir
=
recipe_dir
or
[
os
.
path
.
join
(
os
.
path
.
realpath
(
'.'
),
'conda'
)]
logger
.
info
(
'This package is considered part of group "%s" - tunning '
\
'conda package and documentation URLs for this...'
,
group
)
# get potential channel upload and other auxiliary channels
channels
=
get_channels
(
public
=
(
not
private
),
stable
=
stable
,
server
=
server
,
intranet
=
ci
)
intranet
=
ci
,
group
=
group
)
if
condarc
is
not
None
:
logger
.
info
(
'Loading CONDARC file from %s...'
,
condarc
)
...
...
@@ -117,7 +122,7 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
# and derived documentation building via Sphinx)
set_environment
(
'DOCSERVER'
,
server
)
doc_urls
=
get_docserver_setup
(
public
=
(
not
private
),
stable
=
stable
,
server
=
server
,
intranet
=
ci
)
server
=
server
,
intranet
=
ci
,
group
=
group
)
set_environment
(
'BOB_DOCUMENTATION_SERVER'
,
doc_urls
)
for
d
in
recipe_dir
:
...
...
bob/devtools/scripts/ci.py
View file @
a6eb8531
...
...
@@ -317,6 +317,9 @@ Examples:
'''
)
@
click
.
argument
(
'order'
,
required
=
True
,
type
=
click
.
Path
(
file_okay
=
True
,
dir_okay
=
False
,
exists
=
True
),
nargs
=
1
)
@
click
.
option
(
'-g'
,
'--group'
,
show_default
=
True
,
default
=
os
.
environ
[
'CI_PROJECT_NAMESPACE'
],
help
=
'Group of packages (gitlab namespace) this package belongs to'
)
@
click
.
option
(
'-p'
,
'--python'
,
multiple
=
True
,
help
=
'Versions of python in the format "x.y" we should build for. Pass '
\
'various times this option to build for multiple python versions'
)
...
...
@@ -326,7 +329,7 @@ Examples:
'printing to help you understand what will be done'
)
@
verbosity_option
()
@
bdt
.
raise_on_error
def
base_build
(
order
,
python
,
dry_run
):
def
base_build
(
order
,
group
,
python
,
dry_run
):
"""Builds base (dependence) packages
This command builds dependence packages (packages that are not Bob/BEAT
...
...
@@ -335,7 +338,6 @@ def base_build(order, python, dry_run):
"""
from
..constants
import
CONDA_BUILD_CONFIG
from
..build
import
base_build
as
_build
condarc
=
os
.
path
.
join
(
os
.
environ
[
'CONDA_ROOT'
],
'condarc'
)
logger
.
info
(
'Loading (this build
\'
s) CONDARC file from %s...'
,
condarc
)
...
...
@@ -355,6 +357,7 @@ def base_build(order, python, dry_run):
import
itertools
from
..
import
bootstrap
from
..build
import
base_build
as
_build
# combine all versions of python with recipes
if
python
:
...
...
@@ -371,7 +374,7 @@ def base_build(order, python, dry_run):
if
not
os
.
path
.
exists
(
os
.
path
.
join
(
recipe
,
'meta.yaml'
)):
logger
.
info
(
'Ignoring directory "%s" - no meta.yaml found'
%
recipe
)
continue
_build
(
bootstrap
,
SERVER
,
True
,
recipe
,
CONDA_BUILD_CONFIG
,
pyver
,
_build
(
bootstrap
,
SERVER
,
True
,
group
,
recipe
,
CONDA_BUILD_CONFIG
,
pyver
,
condarc_options
)
...
...
@@ -399,6 +402,11 @@ def test(ctx, dry_run):
from
..constants
import
CONDA_BUILD_CONFIG
,
CONDA_RECIPE_APPEND
group
=
os
.
environ
[
'CI_PROJECT_NAMESPACE'
]
if
group
not
in
(
'bob'
,
'beat'
):
# defaults back to bob - no other server setups are available as of now
group
=
'bob'
from
.test
import
test
ctx
.
invoke
(
test
,
package
=
glob
.
glob
(
os
.
path
.
join
(
os
.
environ
[
'CONDA_ROOT'
],
'conda-bld'
,
...
...
@@ -407,6 +415,7 @@ def test(ctx, dry_run):
config
=
CONDA_BUILD_CONFIG
,
append_file
=
CONDA_RECIPE_APPEND
,
server
=
SERVER
,
group
=
group
,
private
=
(
os
.
environ
[
'CI_PROJECT_VISIBILITY'
]
!=
'public'
),
stable
=
'CI_COMMIT_TAG'
in
os
.
environ
,
dry_run
=
dry_run
,
...
...
@@ -438,6 +447,11 @@ def build(ctx, dry_run):
from
..constants
import
CONDA_BUILD_CONFIG
,
CONDA_RECIPE_APPEND
group
=
os
.
environ
[
'CI_PROJECT_NAMESPACE'
]
if
group
not
in
(
'bob'
,
'beat'
):
# defaults back to bob - no other server setups are available as of now
group
=
'bob'
from
.build
import
build
ctx
.
invoke
(
build
,
recipe_dir
=
[
os
.
path
.
join
(
os
.
path
.
realpath
(
os
.
curdir
),
'conda'
)],
...
...
@@ -447,6 +461,7 @@ def build(ctx, dry_run):
no_test
=
False
,
append_file
=
CONDA_RECIPE_APPEND
,
server
=
SERVER
,
group
=
group
,
private
=
(
os
.
environ
[
'CI_PROJECT_VISIBILITY'
]
!=
'public'
),
stable
=
'CI_COMMIT_TAG'
in
os
.
environ
,
dry_run
=
dry_run
,
...
...
bob/devtools/scripts/create.py
View file @
a6eb8531
...
...
@@ -72,9 +72,11 @@ Examples:
@
click
.
option
(
'-a'
,
'--append-file'
,
show_default
=
True
,
default
=
CONDA_RECIPE_APPEND
,
help
=
'overwrites the path leading to '
\
'appended configuration file to use'
)
@
click
.
option
(
'-S'
,
'--server'
,
show_default
=
True
,
default
=
'https://www.idiap.ch/software/bob'
,
help
=
'Server used for '
\
'downloading conda packages and documentation indexes of required packages'
)
@
click
.
option
(
'-S'
,
'--server'
,
show_default
=
True
,
default
=
SERVER
,
help
=
'Server used for downloading conda packages and documentation '
\
'indexes of required packages'
)
@
click
.
option
(
'-g'
,
'--group'
,
show_default
=
True
,
default
=
'bob'
,
help
=
'Group of packages (gitlab namespace) this package belongs to'
)
@
click
.
option
(
'-P'
,
'--private/--no-private'
,
default
=
False
,
help
=
'Set this to **include** private channels on your build - '
\
'you **must** be at Idiap to execute this build in this case - '
\
...
...
@@ -123,6 +125,9 @@ def create(name, recipe_dir, python, overwrite, condarc, use_local, config,
# set some environment variables before continuing
set_environment
(
'DOCSERVER'
,
server
,
os
.
environ
)
logger
.
info
(
'This package is considered part of group "%s" - tunning '
\
'conda package URLs for this...'
,
group
)
if
condarc
is
not
None
:
logger
.
info
(
'Loading CONDARC file from %s...'
,
condarc
)
with
open
(
condarc
,
'rb'
)
as
f
:
...
...
@@ -131,7 +136,7 @@ def create(name, recipe_dir, python, overwrite, condarc, use_local, config,
# use default and add channels
condarc_options
=
yaml
.
load
(
BASE_CONDARC
)
#n.b.: no channels
channels
=
get_channels
(
public
=
(
not
private
),
stable
=
stable
,
server
=
server
,
intranet
=
private
)
intranet
=
private
,
group
=
group
)
condarc_options
[
'channels'
]
=
channels
+
[
'defaults'
]
conda_config
=
make_conda_config
(
config
,
python
,
append_file
,
condarc_options
)
...
...
bob/devtools/scripts/test.py
View file @
a6eb8531
...
...
@@ -49,14 +49,16 @@ Examples:
@
click
.
option
(
'-a'
,
'--append-file'
,
show_default
=
True
,
default
=
CONDA_RECIPE_APPEND
,
help
=
'overwrites the path leading to '
\
'appended configuration file to use'
)
@
click
.
option
(
'-S'
,
'--server'
,
show_default
=
True
,
default
=
'https://www.idiap.ch/software/bob'
,
help
=
'Server used for '
\
'downloading conda packages and documentation indexes of required packages'
)
@
click
.
option
(
'-S'
,
'--server'
,
show_default
=
True
,
default
=
SERVER
,
help
=
'Server used for downloading conda packages and documentation '
\
'indexes of required packages'
)
@
click
.
option
(
'-g'
,
'--group'
,
show_default
=
True
,
default
=
'bob'
,
help
=
'Group of packages (gitlab namespace) this package belongs to'
)
@
click
.
option
(
'-P'
,
'--private/--no-private'
,
default
=
False
,
help
=
'Set this to **include** private channels on your build - '
\
'you **must** be at Idiap to execute this build in this case - '
\
'you **must** also use the correct server name through --server - '
\
'notice this option has no effect if you also pass --condarc'
)
'notice this option has no effect
to conda
if you also pass --condarc'
)
@
click
.
option
(
'-X'
,
'--stable/--no-stable'
,
default
=
False
,
help
=
'Set this to **exclude** beta channels from your build - '
\
'notice this option has no effect if you also pass --condarc'
)
...
...
@@ -68,7 +70,7 @@ Examples:
help
=
'Use this flag to indicate the build will be running on the CI'
)
@
verbosity_option
()
@
bdt
.
raise_on_error
def
test
(
package
,
condarc
,
config
,
append_file
,
server
,
private
,
stable
,
def
test
(
package
,
condarc
,
config
,
append_file
,
server
,
group
,
private
,
stable
,
dry_run
,
ci
):
"""Tests (pre-built) package through conda-build with stock configuration
...
...
@@ -82,9 +84,12 @@ def test(package, condarc, config, append_file, server, private, stable,
logger
.
warn
(
'!!!! DRY RUN MODE !!!!'
)
logger
.
warn
(
'Nothing will be really built'
)
logger
.
info
(
'This package is considered part of group "%s" - tunning '
\
'conda package and documentation URLs for this...'
,
group
)
# get potential channel upload and other auxiliary channels
channels
=
get_channels
(
public
=
(
not
private
),
stable
=
stable
,
server
=
server
,
intranet
=
ci
)
intranet
=
ci
,
group
=
group
)
if
condarc
is
not
None
:
logger
.
info
(
'Loading CONDARC file from %s...'
,
condarc
)
...
...
@@ -110,7 +115,7 @@ def test(package, condarc, config, append_file, server, private, stable,
# and derived documentation building via Sphinx)
set_environment
(
'DOCSERVER'
,
server
)
doc_urls
=
get_docserver_setup
(
public
=
(
not
private
),
stable
=
stable
,
server
=
server
,
intranet
=
ci
)
server
=
server
,
intranet
=
ci
,
group
=
group
)
set_environment
(
'BOB_DOCUMENTATION_SERVER'
,
doc_urls
)
arch
=
conda_arch
()
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment