diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1ca218d3bf46f1798f288de17bafc93850a5ead8..2b6dd69b2effa39f1b00ac143fd6f10cab65a90d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,11 +14,6 @@ stages:
# Build targets
.build_template:
stage: build
- script:
- - python3 ./bob/devtools/bootstrap.py -vv build
- - source ${CONDA_ROOT}/etc/profile.d/conda.sh
- - conda activate base
- - python3 ./bob/devtools/build.py -vv
artifacts:
expire_in: 1 week
cache:
@@ -26,16 +21,25 @@ stages:
- miniconda.sh
-.build_linux_template:
+build_linux:
extends: .build_template
+ variables:
+ BUILD_EGG: "true"
tags:
- docker
image: continuumio/conda-concourse-ci
before_script:
- rm -f /root/.condarc
- rm -rf /root/.conda
+ script:
+ - python3 ./bob/devtools/bootstrap.py -vv build
+ - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+ - conda activate base
+ - python3 ./bob/devtools/build.py -vv --twine-check
artifacts:
paths:
+ - dist/*.zip
+ - sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
@@ -44,64 +48,23 @@ stages:
key: "linux-cache"
-.build_macosx_template:
+build_macosx:
extends: .build_template
tags:
- macosx
- artifacts:
- paths:
- - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- - ${CONDA_ROOT}/conda-bld/noarch/*.conda
- - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
- - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
- cache:
- key: "macosx-cache"
-
-
-build_linux_36:
- extends: .build_linux_template
- variables:
- PYTHON_VERSION: "3.6"
-
-build_linux_37:
- extends: .build_linux_template
- variables:
- PYTHON_VERSION: "3.7"
-
-build_linux_38:
- extends: .build_linux_template
- variables:
- PYTHON_VERSION: "3.8"
- BUILD_EGG: "true"
script:
- python3 ./bob/devtools/bootstrap.py -vv build
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- - python3 ./bob/devtools/build.py -vv --twine-check
+ - python3 ./bob/devtools/build.py -vv
artifacts:
paths:
- - dist/*.zip
- - sphinx
- - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
+ - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
+ - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
-
-
-build_macosx_36:
- extends: .build_macosx_template
- variables:
- PYTHON_VERSION: "3.6"
-
-build_macosx_37:
- extends: .build_macosx_template
- variables:
- PYTHON_VERSION: "3.7"
-
-build_macosx_38:
- extends: .build_macosx_template
- variables:
- PYTHON_VERSION: "3.8"
+ cache:
+ key: "macosx-cache"
# Deploy targets
@@ -118,12 +81,8 @@ build_macosx_38:
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
- - build_linux_36
- - build_linux_37
- - build_linux_38
- - build_macosx_36
- - build_macosx_37
- - build_macosx_38
+ - build_linux
+ - build_macosx
tags:
- docker
cache:
@@ -164,12 +123,8 @@ pypi:
- bdt ci pypi -vv dist/*.zip
- bdt ci clean -vv
dependencies:
- - build_linux_36
- - build_linux_37
- - build_linux_38
- - build_macosx_36
- - build_macosx_37
- - build_macosx_38
+ - build_linux
+ - build_macosx
tags:
- docker
cache:
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index 93ca35625668ee9f20bbf71f7693096878cdcd49..1b6492034ef329e190f22d6f83f92147c78aaef1 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -102,7 +102,9 @@ def next_build_number(channel_url, basename):
remove_conda_loggers()
# get the channel index
- channel_urls = calculate_channel_urls([channel_url], prepend=False, use_local=False)
+ channel_urls = calculate_channel_urls(
+ [channel_url], prepend=False, use_local=False
+ )
logger.debug("Downloading channel index from %s", channel_urls)
index = fetch_index(channel_urls=channel_urls)
@@ -215,7 +217,7 @@ def make_conda_config(config, python, append_file, condarc_options):
def get_output_path(metadata, config):
"""Renders the recipe and returns the name of the output file."""
- return conda_build.api.get_output_file_paths(metadata, config=config)[0]
+ return conda_build.api.get_output_file_paths(metadata, config=config)
def get_rendered_metadata(recipe_dir, config):
@@ -565,7 +567,6 @@ def base_build(
group,
recipe_dir,
conda_build_config,
- python_version,
condarc_options,
):
"""Builds a non-beat/non-bob software dependence that doesn't exist on
@@ -590,11 +591,6 @@ def base_build(
our internal webserver. Currently, only "bob" or "beat" will work.
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use
- python_version: String with the python version to build for, in the format
- ``x.y`` (should be passed even if not building a python package). It
- can also be set to ``noarch``, or ``None``. If set to ``None``, then we
- don't assume there is a python-specific version being built. If set to
- ``noarch``, then it is a python package without a specific build.
condarc_options: Pre-parsed condarc options loaded from the respective YAML
file
@@ -607,8 +603,7 @@ def base_build(
# if you get to this point, tries to build the package
channels = bootstrap.get_channels(
- public=True, stable=True, server=server, intranet=intranet,
- group=group
+ public=True, stable=True, server=server, intranet=intranet, group=group
)
if "channels" not in condarc_options:
@@ -619,54 +614,40 @@ def base_build(
"\n - ".join(condarc_options["channels"]),
)
logger.info("Merging conda configuration files...")
- if python_version not in ("noarch", None):
- conda_config = make_conda_config(
- conda_build_config, python_version, None, condarc_options
- )
- else:
- conda_config = make_conda_config(
- conda_build_config, None, None, condarc_options
- )
+ conda_config = make_conda_config(
+ conda_build_config, None, None, condarc_options
+ )
metadata = get_rendered_metadata(recipe_dir, conda_config)
-
- # handles different cases as explained on the description of
- # ``python_version``
- py_ver = python_version.replace(".", "") if python_version else None
- if py_ver == "noarch":
- py_ver = ""
arch = conda_arch()
# checks we should actually build this recipe
if should_skip_build(metadata):
- if py_ver is None:
- logger.warn(
- 'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch
- )
- elif not py_ver:
- logger.warn(
- 'Skipping UNSUPPORTED build of "%s" for (noarch) python '
- "on %s",
- recipe_dir,
- arch,
- )
- else:
- logger.warn(
- 'Skipping UNSUPPORTED build of "%s" for python-%s ' "on %s",
- recipe_dir,
- python_version,
- arch,
- )
+ logger.warn(
+ 'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch
+ )
return
- path = get_output_path(metadata, conda_config)
+ paths = get_output_path(metadata, conda_config)
+ urls = [exists_on_channel(channels[0], os.path.basename(k)) for k in paths]
- url = exists_on_channel(channels[0], os.path.basename(path))
- if url is not None:
- logger.info("Skipping build for %s as it exists (at %s)", path, url)
+ if all(urls):
+ logger.info(
+ "Skipping build(s) for recipe at '%s' as packages with matching "
+ "characteristics exist (%s)",
+ recipe_dir,
+ ", ".join(urls),
+ )
return
- # if you get to this point, just builds the package
+ if any(urls):
+ raise RuntimeError(
+ "One or more packages for recipe at '%s' already exist (%s). "
+ "Change the package build number to trigger a build." % \
+ (recipe_dir, ", ".join(urls)),
+ )
+
+ # if you get to this point, just builds the package(s)
logger.info("Building %s", path)
return conda_build.api.build(recipe_dir, config=conda_config)
@@ -718,14 +699,6 @@ if __name__ == "__main__":
default=os.environ.get("CI_PROJECT_DIR", os.path.realpath(os.curdir)),
help="The directory where the repo was cloned [default: %(default)s]",
)
- parser.add_argument(
- "-p",
- "--python-version",
- default=os.environ.get(
- "PYTHON_VERSION", "%d.%d" % sys.version_info[:2]
- ),
- help="The version of python to build for [default: %(default)s]",
- )
parser.add_argument(
"-T",
"--twine-check",
@@ -783,8 +756,9 @@ if __name__ == "__main__":
bootstrap.set_environment("BOB_PACKAGE_VERSION", version)
# create the build configuration
- conda_build_config = os.path.join(mydir, "data", "conda_build_config.yaml")
- recipe_append = os.path.join(mydir, "data", "recipe_append.yaml")
+ conda_build_config = os.path.join(args.work_dir, "conda",
+ "conda_build_config.yaml")
+ recipe_append = os.path.join(args.work_dir, "data", "recipe_append.yaml")
condarc = os.path.join(args.conda_root, "condarc")
logger.info("Loading (this build's) CONDARC file from %s...", condarc)
@@ -812,11 +786,10 @@ if __name__ == "__main__":
args.group,
recipe,
conda_build_config,
- args.python_version,
condarc_options,
)
- public = (args.visibility == "public")
+ public = args.visibility == "public"
channels = bootstrap.get_channels(
public=public,
stable=(not is_prerelease),
@@ -834,36 +807,34 @@ if __name__ == "__main__":
)
logger.info("Merging conda configuration files...")
conda_config = make_conda_config(
- conda_build_config, args.python_version, recipe_append, condarc_options
+ conda_build_config, None, recipe_append, condarc_options
)
recipe_dir = os.path.join(args.work_dir, "conda")
metadata = get_rendered_metadata(recipe_dir, conda_config)
- path = get_output_path(metadata, conda_config)
+ paths = get_output_path(metadata, conda_config)
# asserts we're building at the right location
- assert path.startswith(os.path.join(args.conda_root, "conda-bld")), (
- 'Output path for build (%s) does not start with "%s" - this '
- "typically means this build is running on a shared builder and "
- "the file ~/.conda/environments.txt is polluted with other "
- "environment paths. To fix, empty that file and set its mode "
- "to read-only for all."
- % (path, os.path.join(args.conda_root, "conda-bld"))
- )
+ for path in paths:
+ assert path.startswith(os.path.join(args.conda_root, "conda-bld")), (
+ 'Output path for build (%s) does not start with "%s" - this '
+ "typically means this build is running on a shared builder and "
+ "the file ~/.conda/environments.txt is polluted with other "
+ "environment paths. To fix, empty that file and set its mode "
+ "to read-only for all."
+ % (path, os.path.join(args.conda_root, "conda-bld"))
+ )
+
+ # retrieve the current build number(s) for this build
+ build_numbers = [
+ next_build_number(channels[0], os.path.basename(k))[0] for k in paths
+ ]
- # retrieve the current build number for this build
- build_number, _ = next_build_number(channels[0], os.path.basename(path))
+ # homogenize to the largest build number
+ build_number = max([int(k) for k in build_numbers])
# runs the build using the conda-build API
arch = conda_arch()
- logger.info(
- "Building %s-%s-py%s (build: %d) for %s",
- args.name,
- version,
- args.python_version.replace(".", ""),
- build_number,
- arch,
- )
# notice we cannot build from the pre-parsed metadata because it has already
# resolved the "wrong" build number. We'll have to reparse after setting the
diff --git a/bob/devtools/graph.py b/bob/devtools/graph.py
index c7fe00912ed1ce94c78e7663d316c1248e11be47..a6b7296e627f2f3003973e516e84aedbaa7f3ea5 100644
--- a/bob/devtools/graph.py
+++ b/bob/devtools/graph.py
@@ -124,7 +124,7 @@ def compute_adjencence_matrix(
# pre-renders the recipe - figures out the destination
metadata = get_rendered_metadata(recipe_dir, conda_config)
rendered_recipe = get_parsed_recipe(metadata)
- path = get_output_path(metadata, conda_config)
+ path = get_output_path(metadata, conda_config)[0]
# gets the next build number
build_number, _ = next_build_number(
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index 9bc98d4566b7959b82e77ff72e7732547bfbee76..4601c46b7650e0f0f1320ff51ad276d4d4017b76 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -266,7 +266,7 @@ def build(
continue
rendered_recipe = get_parsed_recipe(metadata)
- path = get_output_path(metadata, conda_config)
+ path = get_output_path(metadata, conda_config)[0]
# gets the next build number
build_number, _ = next_build_number(channels[0], os.path.basename(path))
diff --git a/bob/devtools/scripts/local.py b/bob/devtools/scripts/local.py
index d7bacd6a9598e0bcf080832852dcaff69a1469ce..a008757e12b7e8935f21d534476b5e4ec2271570 100644
--- a/bob/devtools/scripts/local.py
+++ b/bob/devtools/scripts/local.py
@@ -176,13 +176,6 @@ Examples:
"(combine with the verbosity flags - e.g. ``-vvv``) to enable "
"printing to help you understand what will be done",
)
-@click.option(
- "-p",
- "--python",
- multiple=True,
- help='Versions of python in the format "x.y" we should build for. Pass '
- "various times this option to build for multiple python versions",
-)
@click.option(
"-g",
"--group",
@@ -196,7 +189,4 @@ Examples:
def base_build(ctx, order, dry_run, python, group):
"""Run the CI build step locally."""
set_up_environment_variables(python=python, name_space=group)
-
- ctx.invoke(
- ci.base_build, order=order, dry_run=dry_run, group=group, python=python
- )
+ ctx.invoke(ci.base_build, order=order, dry_run=dry_run, group=group)
diff --git a/bob/devtools/scripts/rebuild.py b/bob/devtools/scripts/rebuild.py
index c7bb655ceda3175d3fd6d6c4650452ee0ceff68e..8b5a33710b04795ddbe99c8f4ec59e493ce7c169 100644
--- a/bob/devtools/scripts/rebuild.py
+++ b/bob/devtools/scripts/rebuild.py
@@ -253,7 +253,7 @@ def rebuild(
continue
rendered_recipe = get_parsed_recipe(metadata)
- path = get_output_path(metadata, conda_config)
+ path = get_output_path(metadata, conda_config)[0]
# Get the latest build number
build_number, existing = next_build_number(
diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a4c56ab1cd521d843cba736f4eb0ee212f14943d
--- /dev/null
+++ b/conda/conda_build_config.yaml
@@ -0,0 +1,24 @@
+macos_min_version:
+ - 10.9
+macos_machine:
+ - x86_64-apple-darwin13.4.0
+MACOSX_DEPLOYMENT_TARGET:
+ - 10.9
+CONDA_BUILD_SYSROOT: # [osx]
+ - /opt/MacOSX10.9.sdk # [osx]
+# This helps CMAKE find the sysroot. See
+# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
+SDKROOT: # [osx]
+ - /opt/MacOSX10.9.sdk # [osx]
+# makes autotools verbose
+VERBOSE_AT:
+ - V=1
+# makes cmake verbose
+VERBOSE_CM:
+ - VERBOSE=1
+
+## the dependencies that we build against multiple versions
+python:
+ - 3.6
+ - 3.7
+ - 3.8
diff --git a/conda/meta.yaml b/conda/meta.yaml
index a41ac8893c86bacf1701ce5386d6b4bae861dfb2..3baf01d7f17936bd7a15ab1d29190c81b795fcf2 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -12,10 +12,10 @@ build:
- {{ pin_subpackage(name) }}
script:
- cd {{ environ.get('RECIPE_DIR') + '/..' }}
- {% if environ.get('BUILD_EGG') %}
+ {% if environ.get('BUILD_EGG') and not os.path.exists('dist') %}
- python setup.py sdist --formats=zip
{% endif %}
- - python setup.py install --single-version-externally-managed --record record.txt
+ - {{ PYTHON }} -m pip install --no-deps --ignore-installed .
# installs the documentation source, readme to share/doc so it is available
# during test time
- install -d "${PREFIX}/share/doc/{{ name }}"
@@ -24,7 +24,7 @@ build:
requirements:
host:
- python {{ python }}
- - setuptools {{ setuptools }}
+ - pip
run:
- python
- setuptools
@@ -108,7 +108,9 @@ test:
- bdt gitlab graph --help
- bdt gitlab badges --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
+ {% if not os.path.exists('sphinx') %}
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
+ {% endif %}
about:
home: https://www.idiap.ch/software/bob/
diff --git a/conda/recipe_append.yaml b/conda/recipe_append.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c0481152b266e4b8d412ba2750661527a880c518
--- /dev/null
+++ b/conda/recipe_append.yaml
@@ -0,0 +1,4 @@
+build:
+ script_env:
+ - DOCSERVER
+ - NOSE_EVAL_ATTR