Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.devtools
Commits
6e6b148e
Commit
6e6b148e
authored
Feb 14, 2019
by
André Anjos
💬
Browse files
[build] Support multi-python base builds
parent
e98012f0
Changes
3
Hide whitespace changes
Inline
Side-by-side
bob/devtools/build.py
View file @
6e6b148e
...
...
@@ -86,7 +86,10 @@ def next_build_number(channel_url, name, version, python):
for
dist
in
index
:
if
dist
.
name
==
name
and
dist
.
version
==
version
:
match
=
re
.
match
(
'py[2-9][0-9]+'
,
dist
.
build_string
)
if
py_ver
:
match
=
re
.
match
(
'py[2-9][0-9]+'
,
dist
.
build_string
)
else
:
match
=
re
.
match
(
'py'
,
dist
.
build_string
)
if
match
and
match
.
group
()
==
'py{}'
.
format
(
py_ver
):
logger
.
debug
(
"Found match at %s for %s-%s-py%s"
,
index
[
dist
].
url
,
...
...
@@ -165,7 +168,9 @@ def exists_on_channel(channel_url, name, version, build_number,
name: The name of the package
version: The version of the package
build_number: The build number of the package
python_version: The current version of python we're building for
python_version: The current version of python we're building for. May be
``noarch``, to check for "noarch" packages or ``None``, in which case we
don't check for the python version
Returns: A complete package name, version and build string, if the package
already exists in the channel or ``None`` otherwise.
...
...
@@ -174,8 +179,10 @@ def exists_on_channel(channel_url, name, version, build_number,
from
conda.exports
import
get_index
# no dot in py_ver
py_ver
=
python_version
.
replace
(
'.'
,
''
)
# handles different cases as explained on the description of
# ``python_version``
py_ver
=
python_version
.
replace
(
'.'
,
''
)
if
python_version
else
None
if
py_ver
==
'noarch'
:
py_ver
=
''
# get the channel index
logger
.
debug
(
'Downloading channel index from %s'
,
channel_url
)
...
...
@@ -204,8 +211,12 @@ def exists_on_channel(channel_url, name, version, build_number,
dist
.
version
,
dist
.
build_string
)
return
(
dist
.
name
,
dist
.
version
,
dist
.
build_string
)
logger
.
info
(
'No matches for %s-%s-(py%s_?)%s found among %d packages'
,
name
,
version
,
py_ver
,
build_number
,
len
(
index
))
if
py_ver
is
None
:
logger
.
info
(
'No matches for %s-%s-%s found among %d packages'
,
name
,
version
,
build_number
,
len
(
index
))
else
:
logger
.
info
(
'No matches for %s-%s-py%s_%s found among %d packages'
,
name
,
version
,
py_ver
,
build_number
,
len
(
index
))
return
...
...
@@ -483,7 +494,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use
python_version: String with the python version to build for, in the format
``x.y`` (should be passed even if not building a python package)
``x.y`` (should be passed even if not building a python package). It
can also be set to ``noarch``, or ``None``. If set to ``None``, then we
don't assume there is a python-specific version being built. If set to
``noarch``, then it is a python package without a specific build.
condarc_options: Pre-parsed condarc options loaded from the respective YAML
file
...
...
@@ -498,8 +512,12 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
condarc_options
[
'channels'
]
=
public_channels
+
[
'defaults'
]
logger
.
info
(
'Merging conda configuration files...'
)
conda_config
=
make_conda_config
(
conda_build_config
,
python_version
,
None
,
condarc_options
)
if
python_version
not
in
(
'noarch'
,
None
):
conda_config
=
make_conda_config
(
conda_build_config
,
python_version
,
None
,
condarc_options
)
else
:
conda_config
=
make_conda_config
(
conda_build_config
,
None
,
None
,
condarc_options
)
metadata
=
get_rendered_metadata
(
recipe_dir
,
conda_config
)
recipe
=
get_parsed_recipe
(
metadata
)
...
...
@@ -508,8 +526,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
logger
.
info
(
'Skipping build for %s - rendering returned None'
,
recipe_dir
)
return
# no dot in py_ver
py_ver
=
python_version
.
replace
(
'.'
,
''
)
# handles different cases as explained on the description of
# ``python_version``
py_ver
=
python_version
.
replace
(
'.'
,
''
)
if
python_version
else
None
if
py_ver
==
'noarch'
:
py_ver
=
''
arch
=
conda_arch
()
candidate
=
exists_on_channel
(
public_channels
[
0
],
recipe
[
'package'
][
'name'
],
...
...
@@ -521,9 +541,14 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
return
# if you get to this point, just builds the package
logger
.
info
(
'Building %s-%s-(py%s_?)%s for %s'
,
if
py_ver
is
None
:
logger
.
info
(
'Building %s-%s-%s for %s'
,
recipe
[
'package'
][
'name'
],
recipe
[
'package'
][
'version'
],
recipe
[
'build'
][
'number'
],
py_ver
,
arch
)
recipe
[
'build'
][
'number'
],
arch
)
else
:
logger
.
info
(
'Building %s-%s-py%s_%s for %s'
,
recipe
[
'package'
][
'name'
],
recipe
[
'package'
][
'version'
],
py_ver
,
recipe
[
'build'
][
'number'
],
arch
)
conda_build
.
api
.
build
(
recipe_dir
,
config
=
conda_config
)
...
...
bob/devtools/data/gitlab-ci/base-build.yaml
View file @
6e6b148e
...
...
@@ -21,7 +21,8 @@ stages:
-
python3 bootstrap.py -vv channel base
-
source ${CONDA_ROOT}/etc/profile.d/conda.sh
-
conda activate base
-
bdt ci base-build -vv order.txt
-
bdt ci base-build -vv nopython.txt
-
bdt ci base-build -vv --python=3.6 python.txt
-
'
[
"${CI_COMMIT_REF_NAME}"
=
"master"
]
&&
bdt
ci
base-deploy
-vv'
-
bdt ci clean -vv
cache
:
&build_caches
...
...
@@ -31,10 +32,8 @@ stages:
-
${CONDA_ROOT}/pkgs/urls.txt
build_linux
_36
:
build_linux
:
<<
:
*build_job
variables
:
PYTHON_VERSION
:
"
3.6"
tags
:
-
docker
image
:
continuumio/conda-concourse-ci
...
...
@@ -42,10 +41,9 @@ build_linux_36:
<<
:
*build_caches
key
:
"
linux-cache"
build_macosx_36
:
build_macosx
:
<<
:
*build_job
variables
:
PYTHON_VERSION
:
"
3.6"
tags
:
-
macosx
cache
:
...
...
bob/devtools/scripts/ci.py
View file @
6e6b148e
...
...
@@ -310,13 +310,16 @@ Examples:
'''
)
@
click
.
argument
(
'order'
,
required
=
True
,
type
=
click
.
Path
(
file_okay
=
True
,
dir_okay
=
False
,
exists
=
True
),
nargs
=
1
)
@
click
.
option
(
'-p'
,
'--python'
,
multiple
=
True
,
help
=
'Versions of python in the format "x.y" we should build for. Pass '
\
'various times this option to build for multiple python versions'
)
@
click
.
option
(
'-d'
,
'--dry-run/--no-dry-run'
,
default
=
False
,
help
=
'Only goes through the actions, but does not execute them '
\
'(combine with the verbosity flags - e.g. ``-vvv``) to enable '
\
'printing to help you understand what will be done'
)
@
verbosity_option
()
@
bdt
.
raise_on_error
def
base_build
(
order
,
dry_run
):
def
base_build
(
order
,
python
,
dry_run
):
"""Builds base (dependence) packages
This command builds dependence packages (packages that are not Bob/BEAT
...
...
@@ -343,17 +346,26 @@ def base_build(order, dry_run):
line
=
line
.
partition
(
'#'
)[
0
].
strip
()
if
line
:
recipes
.
append
(
line
)
import
itertools
from
..
import
bootstrap
for
order
,
recipe
in
enumerate
(
recipes
):
click
.
echo
(
'
\n
'
+
(
60
*
'='
))
click
.
echo
(
'Building "%s" (%d/%d)'
%
(
recipe
,
order
+
1
,
len
(
recipes
)))
click
.
echo
((
60
*
'='
)
+
'
\n
'
)
# combine all versions of python with recipes
if
python
:
recipes
=
list
(
itertools
.
product
(
python
,
recipes
))
else
:
recipes
=
list
(
itertools
.
product
([
None
],
recipes
))
for
order
,
(
pyver
,
recipe
)
in
enumerate
(
recipes
):
click
.
echo
(
'
\n
'
+
(
80
*
'='
))
pytext
=
'for python-%s'
%
pyver
if
pyver
is
not
None
else
''
click
.
echo
(
'Building "%s" %s(%d/%d)'
%
\
(
recipe
,
pytext
,
order
+
1
,
total_recipes
))
click
.
echo
((
80
*
'='
)
+
'
\n
'
)
if
not
os
.
path
.
exists
(
os
.
path
.
join
(
recipe
,
'meta.yaml'
)):
logger
.
info
(
'Ignoring directory "%s" - no meta.yaml found'
%
recipe
)
continue
_build
(
bootstrap
,
SERVER
,
True
,
recipe
,
CONDA_BUILD_CONFIG
,
os
.
environ
[
'PYTHON_VERSION'
],
condarc_options
)
_build
(
bootstrap
,
SERVER
,
True
,
recipe
,
CONDA_BUILD_CONFIG
,
pyver
,
condarc_options
)
@
ci
.
command
(
epilog
=
'''
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment