Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.devtools
Commits
4f438f9b
Commit
4f438f9b
authored
Feb 18, 2019
by
André Anjos
💬
Browse files
[scripts][test] New functionality to run test-only builds using conda packages
parent
f4bc6f1d
Pipeline
#27186
canceled with stage
in 9 minutes and 8 seconds
Changes
6
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
bob/devtools/data/gitlab-ci/single-package.yaml
View file @
4f438f9b
...
...
@@ -86,6 +86,42 @@ build_macosx_36:
PYTHON_VERSION
:
"
3.6"
# Test targets (not normally used)
.test_template
:
&test_job
stage
:
test
script
:
-
curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
-
python3 bootstrap.py -vv channel base
-
source ${CONDA_ROOT}/etc/profile.d/conda.sh
-
conda activate base
-
bdt ci test -vv
-
bdt ci clean -vv
cache
:
&test_caches
paths
:
-
miniconda.sh
-
${CONDA_ROOT}/pkgs/*.tar.bz2
-
${CONDA_ROOT}/pkgs/urls.txt
.test_linux_template
:
&linux_test_job
<<
:
*test_job
tags
:
-
docker
image
:
continuumio/conda-concourse-ci
cache
:
<<
:
*test_caches
key
:
"
linux-cache"
.test_macosx_template
:
&macosx_test_job
<<
:
*test_job
tags
:
-
macosx
cache
:
<<
:
*test_caches
key
:
"
macosx-cache"
# Deploy targets
.deploy_template
:
&deploy_job
stage
:
deploy
...
...
bob/devtools/scripts/build.py
View file @
4f438f9b
...
...
@@ -80,9 +80,6 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
This command wraps the execution of conda-build so that you use the same
conda configuration we use for our CI. It always set
``--no-anaconda-upload``.
Note that both files are embedded within bob.devtools - you may need to
update your environment before trying this.
"""
# if we are in a dry-run mode, let's let it be known
...
...
bob/devtools/scripts/ci.py
View file @
4f438f9b
...
...
@@ -375,6 +375,45 @@ def base_build(order, python, dry_run):
condarc_options
)
@
ci
.
command
(
epilog
=
'''
Examples:
1. Tests the current package
$ bdt ci test -vv
'''
)
@
click
.
option
(
'-d'
,
'--dry-run/--no-dry-run'
,
default
=
False
,
help
=
'Only goes through the actions, but does not execute them '
\
'(combine with the verbosity flags - e.g. ``-vvv``) to enable '
\
'printing to help you understand what will be done'
)
@
verbosity_option
()
@
bdt
.
raise_on_error
@
click
.
pass_context
def
test
(
ctx
,
dry_run
):
"""Tests packages
This command tests packages in the CI infrastructure. It is **not** meant
to be used outside this context.
"""
from
..constants
import
CONDA_BUILD_CONFIG
,
CONDA_RECIPE_APPEND
from
.test
import
test
ctx
.
invoke
(
test
,
package
=
glob
.
glob
(
os
.
path
.
join
(
os
.
environ
[
'CONDA_ROOT'
],
'conda-bld'
,
arch
,
name
+
'*.tar.bz2'
)),
condarc
=
None
,
#custom build configuration
config
=
CONDA_BUILD_CONFIG
,
append_file
=
CONDA_RECIPE_APPEND
,
server
=
SERVER
,
private
=
(
os
.
environ
[
'CI_PROJECT_VISIBILITY'
]
!=
'public'
),
stable
=
'CI_COMMIT_TAG'
in
os
.
environ
,
dry_run
=
dry_run
,
ci
=
True
,
)
@
ci
.
command
(
epilog
=
'''
Examples:
...
...
bob/devtools/scripts/test.py
0 → 100644
View file @
4f438f9b
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import
os
import
sys
import
yaml
import
click
import
pkg_resources
import
conda_build.api
from
.
import
bdt
from
..build
import
conda_arch
,
make_conda_config
,
get_docserver_setup
,
\
get_env_directory
from
..constants
import
CONDA_BUILD_CONFIG
,
CONDA_RECIPE_APPEND
,
\
SERVER
,
MATPLOTLIB_RCDIR
,
BASE_CONDARC
from
..bootstrap
import
set_environment
,
get_channels
from
..log
import
verbosity_option
,
get_logger
logger
=
get_logger
(
__name__
)
@
click
.
command
(
epilog
=
'''
Examples:
1. Builds recipe from one of our build dependencies (inside bob.conda):
\b
$ cd bob.conda
$ bdt build -vv conda/libblitz
2. Builds recipe from one of our packages, for Python 3.6 (if that is not already the default for you):
$ bdt build --python=3.6 -vv path/to/conda/dir
3. To build multiple recipes, just pass the paths to them:
$ bdt build --python=3.6 -vv path/to/recipe-dir1 path/to/recipe-dir2
'''
)
@
click
.
argument
(
'package'
,
required
=
True
,
type
=
click
.
Path
(
file_okay
=
True
,
dir_okay
=
False
,
exists
=
True
),
nargs
=-
1
)
@
click
.
option
(
'-r'
,
'--condarc'
,
help
=
'Use custom conda configuration file instead of our own'
,)
@
click
.
option
(
'-m'
,
'--config'
,
'--variant-config-files'
,
show_default
=
True
,
default
=
CONDA_BUILD_CONFIG
,
help
=
'overwrites the path leading to '
\
'variant configuration file to use'
)
@
click
.
option
(
'-a'
,
'--append-file'
,
show_default
=
True
,
default
=
CONDA_RECIPE_APPEND
,
help
=
'overwrites the path leading to '
\
'appended configuration file to use'
)
@
click
.
option
(
'-S'
,
'--server'
,
show_default
=
True
,
default
=
'https://www.idiap.ch/software/bob'
,
help
=
'Server used for '
\
'downloading conda packages and documentation indexes of required packages'
)
@
click
.
option
(
'-P'
,
'--private/--no-private'
,
default
=
False
,
help
=
'Set this to **include** private channels on your build - '
\
'you **must** be at Idiap to execute this build in this case - '
\
'you **must** also use the correct server name through --server - '
\
'notice this option has no effect if you also pass --condarc'
)
@
click
.
option
(
'-X'
,
'--stable/--no-stable'
,
default
=
False
,
help
=
'Set this to **exclude** beta channels from your build - '
\
'notice this option has no effect if you also pass --condarc'
)
@
click
.
option
(
'-d'
,
'--dry-run/--no-dry-run'
,
default
=
False
,
help
=
'Only goes through the actions, but does not execute them '
\
'(combine with the verbosity flags - e.g. ``-vvv``) to enable '
\
'printing to help you understand what will be done'
)
@
click
.
option
(
'-C'
,
'--ci/--no-ci'
,
default
=
False
,
hidden
=
True
,
help
=
'Use this flag to indicate the build will be running on the CI'
)
@
verbosity_option
()
@
bdt
.
raise_on_error
def
test
(
package
,
condarc
,
config
,
append_file
,
server
,
private
,
stable
,
dry_run
,
ci
):
"""Tests (pre-built_ package through conda-build with stock configuration
This command wraps the execution of conda-build so that you use the same
conda configuration we use for our CI. It always set
``--no-anaconda-upload``.
"""
# if we are in a dry-run mode, let's let it be known
if
dry_run
:
logger
.
warn
(
'!!!! DRY RUN MODE !!!!'
)
logger
.
warn
(
'Nothing will be really built'
)
# get potential channel upload and other auxiliary channels
channels
=
get_channels
(
public
=
(
not
private
),
stable
=
stable
,
server
=
server
,
intranet
=
ci
)
if
condarc
is
not
None
:
logger
.
info
(
'Loading CONDARC file from %s...'
,
condarc
)
with
open
(
condarc
,
'rb'
)
as
f
:
condarc_options
=
yaml
.
load
(
f
)
else
:
# use default and add channels
condarc_options
=
yaml
.
load
(
BASE_CONDARC
)
#n.b.: no channels
logger
.
info
(
'Using the following channels during build:
\n
- %s'
,
'
\n
- '
.
join
(
channels
+
[
'defaults'
]))
condarc_options
[
'channels'
]
=
channels
+
[
'defaults'
]
# dump packages at base environment
prefix
=
get_env_directory
(
os
.
environ
[
'CONDA_EXE'
],
'base'
)
condarc_options
[
'croot'
]
=
os
.
path
.
join
(
prefix
,
'conda-bld'
)
conda_config
=
make_conda_config
(
config
,
None
,
append_file
,
condarc_options
)
set_environment
(
'MATPLOTLIBRC'
,
MATPLOTLIB_RCDIR
)
# setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
# and derived documentation building via Sphinx)
set_environment
(
'DOCSERVER'
,
server
)
doc_urls
=
get_docserver_setup
(
public
=
(
not
private
),
stable
=
stable
,
server
=
server
,
intranet
=
ci
)
set_environment
(
'BOB_DOCUMENTATION_SERVER'
,
doc_urls
)
arch
=
conda_arch
()
for
p
in
package
:
logger
.
info
(
'Testing %s at %s'
,
p
,
arch
)
if
not
dry_run
:
conda_build
.
api
.
test
(
p
,
config
=
conda_config
)
conda/meta.yaml
View file @
4f438f9b
...
...
@@ -71,6 +71,7 @@ test:
-
bdt dumpsphinx https://docs.python.org/3/objects.inv > /dev/null
-
bdt create --help
-
bdt build --help
-
bdt test --help
-
bdt getpath --help
#- bdt getpath -vv bob/bob.devtools .gitignore
-
bdt caupdate --help
...
...
setup.py
View file @
4f438f9b
...
...
@@ -55,6 +55,7 @@ setup(
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx'
,
'create = bob.devtools.scripts.create:create'
,
'build = bob.devtools.scripts.build:build'
,
'test = bob.devtools.scripts.test:test'
,
'getpath = bob.devtools.scripts.getpath:getpath'
,
'caupdate = bob.devtools.scripts.caupdate:caupdate'
,
'ci = bob.devtools.scripts.ci:ci'
,
...
...
@@ -63,6 +64,7 @@ setup(
'bdt.ci.cli'
:
[
'base-build = bob.devtools.scripts.ci:base_build'
,
'build = bob.devtools.scripts.ci:build'
,
'test = bob.devtools.scripts.ci:test'
,
'clean = bob.devtools.scripts.ci:clean'
,
'base-deploy = bob.devtools.scripts.ci:base_deploy'
,
'deploy = bob.devtools.scripts.ci:deploy'
,
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment