Commit 08e79373 authored by André Anjos's avatar André Anjos 💬

Merge branch 'proper-multi-build' into 'master'

Fix base_build() with a proper support for multi-package-building through conda-build

See merge request !160
parents e8029310 7ca2788b
Pipeline #40153 passed with stages
in 9 minutes and 31 seconds
...@@ -14,11 +14,6 @@ stages: ...@@ -14,11 +14,6 @@ stages:
# Build targets # Build targets
.build_template: .build_template:
stage: build stage: build
script:
- python3 ./bob/devtools/bootstrap.py -vv build
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- python3 ./bob/devtools/build.py -vv
artifacts: artifacts:
expire_in: 1 week expire_in: 1 week
cache: cache:
...@@ -26,16 +21,25 @@ stages: ...@@ -26,16 +21,25 @@ stages:
- miniconda.sh - miniconda.sh
.build_linux_template: build_linux:
extends: .build_template extends: .build_template
variables:
BUILD_EGG: "true"
tags: tags:
- docker - docker
image: continuumio/conda-concourse-ci image: continuumio/conda-concourse-ci
before_script: before_script:
- rm -f /root/.condarc - rm -f /root/.condarc
- rm -rf /root/.conda - rm -rf /root/.conda
script:
- python3 ./bob/devtools/bootstrap.py -vv build
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- python3 ./bob/devtools/build.py -vv --twine-check
artifacts: artifacts:
paths: paths:
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda - ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
...@@ -44,64 +48,23 @@ stages: ...@@ -44,64 +48,23 @@ stages:
key: "linux-cache" key: "linux-cache"
.build_macosx_template: build_macosx:
extends: .build_template extends: .build_template
tags: tags:
- macosx - macosx
artifacts:
paths:
- ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
cache:
key: "macosx-cache"
build_linux_36:
extends: .build_linux_template
variables:
PYTHON_VERSION: "3.6"
build_linux_37:
extends: .build_linux_template
variables:
PYTHON_VERSION: "3.7"
build_linux_38:
extends: .build_linux_template
variables:
PYTHON_VERSION: "3.8"
BUILD_EGG: "true"
script: script:
- python3 ./bob/devtools/bootstrap.py -vv build - python3 ./bob/devtools/bootstrap.py -vv build
- source ${CONDA_ROOT}/etc/profile.d/conda.sh - source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base - conda activate base
- python3 ./bob/devtools/build.py -vv --twine-check - python3 ./bob/devtools/build.py -vv
artifacts: artifacts:
paths: paths:
- dist/*.zip - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda - ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
cache:
key: "macosx-cache"
build_macosx_36:
extends: .build_macosx_template
variables:
PYTHON_VERSION: "3.6"
build_macosx_37:
extends: .build_macosx_template
variables:
PYTHON_VERSION: "3.7"
build_macosx_38:
extends: .build_macosx_template
variables:
PYTHON_VERSION: "3.8"
# Deploy targets # Deploy targets
...@@ -118,12 +81,8 @@ build_macosx_38: ...@@ -118,12 +81,8 @@ build_macosx_38:
- bdt ci deploy -vv - bdt ci deploy -vv
- bdt ci clean -vv - bdt ci clean -vv
dependencies: dependencies:
- build_linux_36 - build_linux
- build_linux_37 - build_macosx
- build_linux_38
- build_macosx_36
- build_macosx_37
- build_macosx_38
tags: tags:
- docker - docker
cache: cache:
...@@ -164,12 +123,8 @@ pypi: ...@@ -164,12 +123,8 @@ pypi:
- bdt ci pypi -vv dist/*.zip - bdt ci pypi -vv dist/*.zip
- bdt ci clean -vv - bdt ci clean -vv
dependencies: dependencies:
- build_linux_36 - build_linux
- build_linux_37 - build_macosx
- build_linux_38
- build_macosx_36
- build_macosx_37
- build_macosx_38
tags: tags:
- docker - docker
cache: cache:
......
...@@ -102,7 +102,9 @@ def next_build_number(channel_url, basename): ...@@ -102,7 +102,9 @@ def next_build_number(channel_url, basename):
remove_conda_loggers() remove_conda_loggers()
# get the channel index # get the channel index
channel_urls = calculate_channel_urls([channel_url], prepend=False, use_local=False) channel_urls = calculate_channel_urls(
[channel_url], prepend=False, use_local=False
)
logger.debug("Downloading channel index from %s", channel_urls) logger.debug("Downloading channel index from %s", channel_urls)
index = fetch_index(channel_urls=channel_urls) index = fetch_index(channel_urls=channel_urls)
...@@ -215,7 +217,7 @@ def make_conda_config(config, python, append_file, condarc_options): ...@@ -215,7 +217,7 @@ def make_conda_config(config, python, append_file, condarc_options):
def get_output_path(metadata, config): def get_output_path(metadata, config):
"""Renders the recipe and returns the name of the output file.""" """Renders the recipe and returns the name of the output file."""
return conda_build.api.get_output_file_paths(metadata, config=config)[0] return conda_build.api.get_output_file_paths(metadata, config=config)
def get_rendered_metadata(recipe_dir, config): def get_rendered_metadata(recipe_dir, config):
...@@ -565,7 +567,6 @@ def base_build( ...@@ -565,7 +567,6 @@ def base_build(
group, group,
recipe_dir, recipe_dir,
conda_build_config, conda_build_config,
python_version,
condarc_options, condarc_options,
): ):
"""Builds a non-beat/non-bob software dependence that doesn't exist on """Builds a non-beat/non-bob software dependence that doesn't exist on
...@@ -590,11 +591,6 @@ def base_build( ...@@ -590,11 +591,6 @@ def base_build(
our internal webserver. Currently, only "bob" or "beat" will work. our internal webserver. Currently, only "bob" or "beat" will work.
recipe_dir: The directory containing the recipe's ``meta.yaml`` file recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use conda_build_config: Path to the ``conda_build_config.yaml`` file to use
python_version: String with the python version to build for, in the format
``x.y`` (should be passed even if not building a python package). It
can also be set to ``noarch``, or ``None``. If set to ``None``, then we
don't assume there is a python-specific version being built. If set to
``noarch``, then it is a python package without a specific build.
condarc_options: Pre-parsed condarc options loaded from the respective YAML condarc_options: Pre-parsed condarc options loaded from the respective YAML
file file
...@@ -607,8 +603,7 @@ def base_build( ...@@ -607,8 +603,7 @@ def base_build(
# if you get to this point, tries to build the package # if you get to this point, tries to build the package
channels = bootstrap.get_channels( channels = bootstrap.get_channels(
public=True, stable=True, server=server, intranet=intranet, public=True, stable=True, server=server, intranet=intranet, group=group
group=group
) )
if "channels" not in condarc_options: if "channels" not in condarc_options:
...@@ -619,54 +614,40 @@ def base_build( ...@@ -619,54 +614,40 @@ def base_build(
"\n - ".join(condarc_options["channels"]), "\n - ".join(condarc_options["channels"]),
) )
logger.info("Merging conda configuration files...") logger.info("Merging conda configuration files...")
if python_version not in ("noarch", None): conda_config = make_conda_config(
conda_config = make_conda_config( conda_build_config, None, None, condarc_options
conda_build_config, python_version, None, condarc_options )
)
else:
conda_config = make_conda_config(
conda_build_config, None, None, condarc_options
)
metadata = get_rendered_metadata(recipe_dir, conda_config) metadata = get_rendered_metadata(recipe_dir, conda_config)
# handles different cases as explained on the description of
# ``python_version``
py_ver = python_version.replace(".", "") if python_version else None
if py_ver == "noarch":
py_ver = ""
arch = conda_arch() arch = conda_arch()
# checks we should actually build this recipe # checks we should actually build this recipe
if should_skip_build(metadata): if should_skip_build(metadata):
if py_ver is None: logger.warn(
logger.warn( 'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch
'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch )
)
elif not py_ver:
logger.warn(
'Skipping UNSUPPORTED build of "%s" for (noarch) python '
"on %s",
recipe_dir,
arch,
)
else:
logger.warn(
'Skipping UNSUPPORTED build of "%s" for python-%s ' "on %s",
recipe_dir,
python_version,
arch,
)
return return
path = get_output_path(metadata, conda_config) paths = get_output_path(metadata, conda_config)
urls = [exists_on_channel(channels[0], os.path.basename(k)) for k in paths]
url = exists_on_channel(channels[0], os.path.basename(path)) if all(urls):
if url is not None: logger.info(
logger.info("Skipping build for %s as it exists (at %s)", path, url) "Skipping build(s) for recipe at '%s' as packages with matching "
"characteristics exist (%s)",
recipe_dir,
", ".join(urls),
)
return return
# if you get to this point, just builds the package if any(urls):
raise RuntimeError(
"One or more packages for recipe at '%s' already exist (%s). "
"Change the package build number to trigger a build." % \
(recipe_dir, ", ".join(urls)),
)
# if you get to this point, just builds the package(s)
logger.info("Building %s", path) logger.info("Building %s", path)
return conda_build.api.build(recipe_dir, config=conda_config) return conda_build.api.build(recipe_dir, config=conda_config)
...@@ -718,14 +699,6 @@ if __name__ == "__main__": ...@@ -718,14 +699,6 @@ if __name__ == "__main__":
default=os.environ.get("CI_PROJECT_DIR", os.path.realpath(os.curdir)), default=os.environ.get("CI_PROJECT_DIR", os.path.realpath(os.curdir)),
help="The directory where the repo was cloned [default: %(default)s]", help="The directory where the repo was cloned [default: %(default)s]",
) )
parser.add_argument(
"-p",
"--python-version",
default=os.environ.get(
"PYTHON_VERSION", "%d.%d" % sys.version_info[:2]
),
help="The version of python to build for [default: %(default)s]",
)
parser.add_argument( parser.add_argument(
"-T", "-T",
"--twine-check", "--twine-check",
...@@ -783,8 +756,9 @@ if __name__ == "__main__": ...@@ -783,8 +756,9 @@ if __name__ == "__main__":
bootstrap.set_environment("BOB_PACKAGE_VERSION", version) bootstrap.set_environment("BOB_PACKAGE_VERSION", version)
# create the build configuration # create the build configuration
conda_build_config = os.path.join(mydir, "data", "conda_build_config.yaml") conda_build_config = os.path.join(args.work_dir, "conda",
recipe_append = os.path.join(mydir, "data", "recipe_append.yaml") "conda_build_config.yaml")
recipe_append = os.path.join(args.work_dir, "data", "recipe_append.yaml")
condarc = os.path.join(args.conda_root, "condarc") condarc = os.path.join(args.conda_root, "condarc")
logger.info("Loading (this build's) CONDARC file from %s...", condarc) logger.info("Loading (this build's) CONDARC file from %s...", condarc)
...@@ -812,11 +786,10 @@ if __name__ == "__main__": ...@@ -812,11 +786,10 @@ if __name__ == "__main__":
args.group, args.group,
recipe, recipe,
conda_build_config, conda_build_config,
args.python_version,
condarc_options, condarc_options,
) )
public = (args.visibility == "public") public = args.visibility == "public"
channels = bootstrap.get_channels( channels = bootstrap.get_channels(
public=public, public=public,
stable=(not is_prerelease), stable=(not is_prerelease),
...@@ -834,36 +807,34 @@ if __name__ == "__main__": ...@@ -834,36 +807,34 @@ if __name__ == "__main__":
) )
logger.info("Merging conda configuration files...") logger.info("Merging conda configuration files...")
conda_config = make_conda_config( conda_config = make_conda_config(
conda_build_config, args.python_version, recipe_append, condarc_options conda_build_config, None, recipe_append, condarc_options
) )
recipe_dir = os.path.join(args.work_dir, "conda") recipe_dir = os.path.join(args.work_dir, "conda")
metadata = get_rendered_metadata(recipe_dir, conda_config) metadata = get_rendered_metadata(recipe_dir, conda_config)
path = get_output_path(metadata, conda_config) paths = get_output_path(metadata, conda_config)
# asserts we're building at the right location # asserts we're building at the right location
assert path.startswith(os.path.join(args.conda_root, "conda-bld")), ( for path in paths:
'Output path for build (%s) does not start with "%s" - this ' assert path.startswith(os.path.join(args.conda_root, "conda-bld")), (
"typically means this build is running on a shared builder and " 'Output path for build (%s) does not start with "%s" - this '
"the file ~/.conda/environments.txt is polluted with other " "typically means this build is running on a shared builder and "
"environment paths. To fix, empty that file and set its mode " "the file ~/.conda/environments.txt is polluted with other "
"to read-only for all." "environment paths. To fix, empty that file and set its mode "
% (path, os.path.join(args.conda_root, "conda-bld")) "to read-only for all."
) % (path, os.path.join(args.conda_root, "conda-bld"))
)
# retrieve the current build number(s) for this build
build_numbers = [
next_build_number(channels[0], os.path.basename(k))[0] for k in paths
]
# retrieve the current build number for this build # homogenize to the largest build number
build_number, _ = next_build_number(channels[0], os.path.basename(path)) build_number = max([int(k) for k in build_numbers])
# runs the build using the conda-build API # runs the build using the conda-build API
arch = conda_arch() arch = conda_arch()
logger.info(
"Building %s-%s-py%s (build: %d) for %s",
args.name,
version,
args.python_version.replace(".", ""),
build_number,
arch,
)
# notice we cannot build from the pre-parsed metadata because it has already # notice we cannot build from the pre-parsed metadata because it has already
# resolved the "wrong" build number. We'll have to reparse after setting the # resolved the "wrong" build number. We'll have to reparse after setting the
......
...@@ -124,7 +124,7 @@ def compute_adjencence_matrix( ...@@ -124,7 +124,7 @@ def compute_adjencence_matrix(
# pre-renders the recipe - figures out the destination # pre-renders the recipe - figures out the destination
metadata = get_rendered_metadata(recipe_dir, conda_config) metadata = get_rendered_metadata(recipe_dir, conda_config)
rendered_recipe = get_parsed_recipe(metadata) rendered_recipe = get_parsed_recipe(metadata)
path = get_output_path(metadata, conda_config) path = get_output_path(metadata, conda_config)[0]
# gets the next build number # gets the next build number
build_number, _ = next_build_number( build_number, _ = next_build_number(
......
...@@ -266,7 +266,7 @@ def build( ...@@ -266,7 +266,7 @@ def build(
continue continue
rendered_recipe = get_parsed_recipe(metadata) rendered_recipe = get_parsed_recipe(metadata)
path = get_output_path(metadata, conda_config) path = get_output_path(metadata, conda_config)[0]
# gets the next build number # gets the next build number
build_number, _ = next_build_number(channels[0], os.path.basename(path)) build_number, _ = next_build_number(channels[0], os.path.basename(path))
......
...@@ -176,13 +176,6 @@ Examples: ...@@ -176,13 +176,6 @@ Examples:
"(combine with the verbosity flags - e.g. ``-vvv``) to enable " "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
"printing to help you understand what will be done", "printing to help you understand what will be done",
) )
@click.option(
"-p",
"--python",
multiple=True,
help='Versions of python in the format "x.y" we should build for. Pass '
"various times this option to build for multiple python versions",
)
@click.option( @click.option(
"-g", "-g",
"--group", "--group",
...@@ -196,7 +189,4 @@ Examples: ...@@ -196,7 +189,4 @@ Examples:
def base_build(ctx, order, dry_run, python, group): def base_build(ctx, order, dry_run, python, group):
"""Run the CI build step locally.""" """Run the CI build step locally."""
set_up_environment_variables(python=python, name_space=group) set_up_environment_variables(python=python, name_space=group)
ctx.invoke(ci.base_build, order=order, dry_run=dry_run, group=group)
ctx.invoke(
ci.base_build, order=order, dry_run=dry_run, group=group, python=python
)
...@@ -253,7 +253,7 @@ def rebuild( ...@@ -253,7 +253,7 @@ def rebuild(
continue continue
rendered_recipe = get_parsed_recipe(metadata) rendered_recipe = get_parsed_recipe(metadata)
path = get_output_path(metadata, conda_config) path = get_output_path(metadata, conda_config)[0]
# Get the latest build number # Get the latest build number
build_number, existing = next_build_number( build_number, existing = next_build_number(
......
macos_min_version:
- 10.9
macos_machine:
- x86_64-apple-darwin13.4.0
MACOSX_DEPLOYMENT_TARGET:
- 10.9
CONDA_BUILD_SYSROOT: # [osx]
- /opt/MacOSX10.9.sdk # [osx]
# This helps CMAKE find the sysroot. See
# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
SDKROOT: # [osx]
- /opt/MacOSX10.9.sdk # [osx]
# makes autotools verbose
VERBOSE_AT:
- V=1
# makes cmake verbose
VERBOSE_CM:
- VERBOSE=1
## the dependencies that we build against multiple versions
python:
- 3.6
- 3.7
- 3.8
...@@ -12,10 +12,10 @@ build: ...@@ -12,10 +12,10 @@ build:
- {{ pin_subpackage(name) }} - {{ pin_subpackage(name) }}
script: script:
- cd {{ environ.get('RECIPE_DIR') + '/..' }} - cd {{ environ.get('RECIPE_DIR') + '/..' }}
{% if environ.get('BUILD_EGG') %} {% if environ.get('BUILD_EGG') and not os.path.exists('dist') %}
- python setup.py sdist --formats=zip - python setup.py sdist --formats=zip
{% endif %} {% endif %}
- python setup.py install --single-version-externally-managed --record record.txt - {{ PYTHON }} -m pip install --no-deps --ignore-installed .
# installs the documentation source, readme to share/doc so it is available # installs the documentation source, readme to share/doc so it is available
# during test time # during test time
- install -d "${PREFIX}/share/doc/{{ name }}" - install -d "${PREFIX}/share/doc/{{ name }}"
...@@ -24,7 +24,7 @@ build: ...@@ -24,7 +24,7 @@ build:
requirements: requirements:
host: host:
- python {{ python }} - python {{ python }}
- setuptools {{ setuptools }} - pip
run: run:
- python - python
- setuptools - setuptools
...@@ -108,7 +108,9 @@ test: ...@@ -108,7 +108,9 @@ test:
- bdt gitlab graph --help - bdt gitlab graph --help
- bdt gitlab badges --help - bdt gitlab badges --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
{% if not os.path.exists('sphinx') %}
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi - if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
{% endif %}
about: about:
home: https://www.idiap.ch/software/bob/ home: https://www.idiap.ch/software/bob/
......
build:
script_env:
- DOCSERVER
- NOSE_EVAL_ATTR
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment