Commit 08e79373 authored by André Anjos's avatar André Anjos 💬

Merge branch 'proper-multi-build' into 'master'

Fix base_build() with a proper support for multi-package-building through conda-build

See merge request !160
parents e8029310 7ca2788b
Pipeline #40153 passed with stages
in 9 minutes and 31 seconds
......@@ -14,11 +14,6 @@ stages:
# Build targets
.build_template:
stage: build
script:
- python3 ./bob/devtools/bootstrap.py -vv build
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- python3 ./bob/devtools/build.py -vv
artifacts:
expire_in: 1 week
cache:
......@@ -26,16 +21,25 @@ stages:
- miniconda.sh
.build_linux_template:
build_linux:
extends: .build_template
variables:
BUILD_EGG: "true"
tags:
- docker
image: continuumio/conda-concourse-ci
before_script:
- rm -f /root/.condarc
- rm -rf /root/.conda
script:
- python3 ./bob/devtools/bootstrap.py -vv build
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- python3 ./bob/devtools/build.py -vv --twine-check
artifacts:
paths:
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
......@@ -44,64 +48,23 @@ stages:
key: "linux-cache"
.build_macosx_template:
build_macosx:
extends: .build_template
tags:
- macosx
artifacts:
paths:
- ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
cache:
key: "macosx-cache"
build_linux_36:
extends: .build_linux_template
variables:
PYTHON_VERSION: "3.6"
build_linux_37:
extends: .build_linux_template
variables:
PYTHON_VERSION: "3.7"
build_linux_38:
extends: .build_linux_template
variables:
PYTHON_VERSION: "3.8"
BUILD_EGG: "true"
script:
- python3 ./bob/devtools/bootstrap.py -vv build
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- python3 ./bob/devtools/build.py -vv --twine-check
- python3 ./bob/devtools/build.py -vv
artifacts:
paths:
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
build_macosx_36:
extends: .build_macosx_template
variables:
PYTHON_VERSION: "3.6"
build_macosx_37:
extends: .build_macosx_template
variables:
PYTHON_VERSION: "3.7"
build_macosx_38:
extends: .build_macosx_template
variables:
PYTHON_VERSION: "3.8"
cache:
key: "macosx-cache"
# Deploy targets
......@@ -118,12 +81,8 @@ build_macosx_38:
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
- build_linux_36
- build_linux_37
- build_linux_38
- build_macosx_36
- build_macosx_37
- build_macosx_38
- build_linux
- build_macosx
tags:
- docker
cache:
......@@ -164,12 +123,8 @@ pypi:
- bdt ci pypi -vv dist/*.zip
- bdt ci clean -vv
dependencies:
- build_linux_36
- build_linux_37
- build_linux_38
- build_macosx_36
- build_macosx_37
- build_macosx_38
- build_linux
- build_macosx
tags:
- docker
cache:
......
......@@ -102,7 +102,9 @@ def next_build_number(channel_url, basename):
remove_conda_loggers()
# get the channel index
channel_urls = calculate_channel_urls([channel_url], prepend=False, use_local=False)
channel_urls = calculate_channel_urls(
[channel_url], prepend=False, use_local=False
)
logger.debug("Downloading channel index from %s", channel_urls)
index = fetch_index(channel_urls=channel_urls)
......@@ -215,7 +217,7 @@ def make_conda_config(config, python, append_file, condarc_options):
def get_output_path(metadata, config):
"""Renders the recipe and returns the name of the output file."""
return conda_build.api.get_output_file_paths(metadata, config=config)[0]
return conda_build.api.get_output_file_paths(metadata, config=config)
def get_rendered_metadata(recipe_dir, config):
......@@ -565,7 +567,6 @@ def base_build(
group,
recipe_dir,
conda_build_config,
python_version,
condarc_options,
):
"""Builds a non-beat/non-bob software dependence that doesn't exist on
......@@ -590,11 +591,6 @@ def base_build(
our internal webserver. Currently, only "bob" or "beat" will work.
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use
python_version: String with the python version to build for, in the format
``x.y`` (should be passed even if not building a python package). It
can also be set to ``noarch``, or ``None``. If set to ``None``, then we
don't assume there is a python-specific version being built. If set to
``noarch``, then it is a python package without a specific build.
condarc_options: Pre-parsed condarc options loaded from the respective YAML
file
......@@ -607,8 +603,7 @@ def base_build(
# if you get to this point, tries to build the package
channels = bootstrap.get_channels(
public=True, stable=True, server=server, intranet=intranet,
group=group
public=True, stable=True, server=server, intranet=intranet, group=group
)
if "channels" not in condarc_options:
......@@ -619,54 +614,40 @@ def base_build(
"\n - ".join(condarc_options["channels"]),
)
logger.info("Merging conda configuration files...")
if python_version not in ("noarch", None):
conda_config = make_conda_config(
conda_build_config, python_version, None, condarc_options
)
else:
conda_config = make_conda_config(
conda_build_config, None, None, condarc_options
)
conda_config = make_conda_config(
conda_build_config, None, None, condarc_options
)
metadata = get_rendered_metadata(recipe_dir, conda_config)
# handles different cases as explained on the description of
# ``python_version``
py_ver = python_version.replace(".", "") if python_version else None
if py_ver == "noarch":
py_ver = ""
arch = conda_arch()
# checks we should actually build this recipe
if should_skip_build(metadata):
if py_ver is None:
logger.warn(
'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch
)
elif not py_ver:
logger.warn(
'Skipping UNSUPPORTED build of "%s" for (noarch) python '
"on %s",
recipe_dir,
arch,
)
else:
logger.warn(
'Skipping UNSUPPORTED build of "%s" for python-%s ' "on %s",
recipe_dir,
python_version,
arch,
)
logger.warn(
'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch
)
return
path = get_output_path(metadata, conda_config)
paths = get_output_path(metadata, conda_config)
urls = [exists_on_channel(channels[0], os.path.basename(k)) for k in paths]
url = exists_on_channel(channels[0], os.path.basename(path))
if url is not None:
logger.info("Skipping build for %s as it exists (at %s)", path, url)
if all(urls):
logger.info(
"Skipping build(s) for recipe at '%s' as packages with matching "
"characteristics exist (%s)",
recipe_dir,
", ".join(urls),
)
return
# if you get to this point, just builds the package
if any(urls):
raise RuntimeError(
"One or more packages for recipe at '%s' already exist (%s). "
"Change the package build number to trigger a build." % \
(recipe_dir, ", ".join(urls)),
)
# if you get to this point, just builds the package(s)
logger.info("Building %s", path)
return conda_build.api.build(recipe_dir, config=conda_config)
......@@ -718,14 +699,6 @@ if __name__ == "__main__":
default=os.environ.get("CI_PROJECT_DIR", os.path.realpath(os.curdir)),
help="The directory where the repo was cloned [default: %(default)s]",
)
parser.add_argument(
"-p",
"--python-version",
default=os.environ.get(
"PYTHON_VERSION", "%d.%d" % sys.version_info[:2]
),
help="The version of python to build for [default: %(default)s]",
)
parser.add_argument(
"-T",
"--twine-check",
......@@ -783,8 +756,9 @@ if __name__ == "__main__":
bootstrap.set_environment("BOB_PACKAGE_VERSION", version)
# create the build configuration
conda_build_config = os.path.join(mydir, "data", "conda_build_config.yaml")
recipe_append = os.path.join(mydir, "data", "recipe_append.yaml")
conda_build_config = os.path.join(args.work_dir, "conda",
"conda_build_config.yaml")
recipe_append = os.path.join(args.work_dir, "data", "recipe_append.yaml")
condarc = os.path.join(args.conda_root, "condarc")
logger.info("Loading (this build's) CONDARC file from %s...", condarc)
......@@ -812,11 +786,10 @@ if __name__ == "__main__":
args.group,
recipe,
conda_build_config,
args.python_version,
condarc_options,
)
public = (args.visibility == "public")
public = args.visibility == "public"
channels = bootstrap.get_channels(
public=public,
stable=(not is_prerelease),
......@@ -834,36 +807,34 @@ if __name__ == "__main__":
)
logger.info("Merging conda configuration files...")
conda_config = make_conda_config(
conda_build_config, args.python_version, recipe_append, condarc_options
conda_build_config, None, recipe_append, condarc_options
)
recipe_dir = os.path.join(args.work_dir, "conda")
metadata = get_rendered_metadata(recipe_dir, conda_config)
path = get_output_path(metadata, conda_config)
paths = get_output_path(metadata, conda_config)
# asserts we're building at the right location
assert path.startswith(os.path.join(args.conda_root, "conda-bld")), (
'Output path for build (%s) does not start with "%s" - this '
"typically means this build is running on a shared builder and "
"the file ~/.conda/environments.txt is polluted with other "
"environment paths. To fix, empty that file and set its mode "
"to read-only for all."
% (path, os.path.join(args.conda_root, "conda-bld"))
)
for path in paths:
assert path.startswith(os.path.join(args.conda_root, "conda-bld")), (
'Output path for build (%s) does not start with "%s" - this '
"typically means this build is running on a shared builder and "
"the file ~/.conda/environments.txt is polluted with other "
"environment paths. To fix, empty that file and set its mode "
"to read-only for all."
% (path, os.path.join(args.conda_root, "conda-bld"))
)
# retrieve the current build number(s) for this build
build_numbers = [
next_build_number(channels[0], os.path.basename(k))[0] for k in paths
]
# retrieve the current build number for this build
build_number, _ = next_build_number(channels[0], os.path.basename(path))
# homogenize to the largest build number
build_number = max([int(k) for k in build_numbers])
# runs the build using the conda-build API
arch = conda_arch()
logger.info(
"Building %s-%s-py%s (build: %d) for %s",
args.name,
version,
args.python_version.replace(".", ""),
build_number,
arch,
)
# notice we cannot build from the pre-parsed metadata because it has already
# resolved the "wrong" build number. We'll have to reparse after setting the
......
......@@ -124,7 +124,7 @@ def compute_adjencence_matrix(
# pre-renders the recipe - figures out the destination
metadata = get_rendered_metadata(recipe_dir, conda_config)
rendered_recipe = get_parsed_recipe(metadata)
path = get_output_path(metadata, conda_config)
path = get_output_path(metadata, conda_config)[0]
# gets the next build number
build_number, _ = next_build_number(
......
......@@ -266,7 +266,7 @@ def build(
continue
rendered_recipe = get_parsed_recipe(metadata)
path = get_output_path(metadata, conda_config)
path = get_output_path(metadata, conda_config)[0]
# gets the next build number
build_number, _ = next_build_number(channels[0], os.path.basename(path))
......
......@@ -176,13 +176,6 @@ Examples:
"(combine with the verbosity flags - e.g. ``-vvv``) to enable "
"printing to help you understand what will be done",
)
@click.option(
"-p",
"--python",
multiple=True,
help='Versions of python in the format "x.y" we should build for. Pass '
"various times this option to build for multiple python versions",
)
@click.option(
"-g",
"--group",
......@@ -196,7 +189,4 @@ Examples:
def base_build(ctx, order, dry_run, python, group):
"""Run the CI build step locally."""
set_up_environment_variables(python=python, name_space=group)
ctx.invoke(
ci.base_build, order=order, dry_run=dry_run, group=group, python=python
)
ctx.invoke(ci.base_build, order=order, dry_run=dry_run, group=group)
......@@ -253,7 +253,7 @@ def rebuild(
continue
rendered_recipe = get_parsed_recipe(metadata)
path = get_output_path(metadata, conda_config)
path = get_output_path(metadata, conda_config)[0]
# Get the latest build number
build_number, existing = next_build_number(
......
macos_min_version:
- 10.9
macos_machine:
- x86_64-apple-darwin13.4.0
MACOSX_DEPLOYMENT_TARGET:
- 10.9
CONDA_BUILD_SYSROOT: # [osx]
- /opt/MacOSX10.9.sdk # [osx]
# This helps CMAKE find the sysroot. See
# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
SDKROOT: # [osx]
- /opt/MacOSX10.9.sdk # [osx]
# makes autotools verbose
VERBOSE_AT:
- V=1
# makes cmake verbose
VERBOSE_CM:
- VERBOSE=1
## the dependencies that we build against multiple versions
python:
- 3.6
- 3.7
- 3.8
......@@ -12,10 +12,10 @@ build:
- {{ pin_subpackage(name) }}
script:
- cd {{ environ.get('RECIPE_DIR') + '/..' }}
{% if environ.get('BUILD_EGG') %}
{% if environ.get('BUILD_EGG') and not os.path.exists('dist') %}
- python setup.py sdist --formats=zip
{% endif %}
- python setup.py install --single-version-externally-managed --record record.txt
- {{ PYTHON }} -m pip install --no-deps --ignore-installed .
# installs the documentation source, readme to share/doc so it is available
# during test time
- install -d "${PREFIX}/share/doc/{{ name }}"
......@@ -24,7 +24,7 @@ build:
requirements:
host:
- python {{ python }}
- setuptools {{ setuptools }}
- pip
run:
- python
- setuptools
......@@ -108,7 +108,9 @@ test:
- bdt gitlab graph --help
- bdt gitlab badges --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
{% if not os.path.exists('sphinx') %}
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
{% endif %}
about:
home: https://www.idiap.ch/software/bob/
......
build:
script_env:
- DOCSERVER
- NOSE_EVAL_ATTR
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment