Merge branch 'proper-multi-build' into 'master'

Fix base_build() with a proper support for multi-package-building through conda-build

See merge request !160

.gitlab-ci.yml

@@ -14,11 +14,6 @@ stages:
 # Build targets
 .build_template:
   stage: build
-  script:
-    - python3 ./bob/devtools/bootstrap.py -vv build
-    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate base
-    - python3 ./bob/devtools/build.py -vv
   artifacts:
     expire_in: 1 week
   cache:
@@ -26,16 +21,25 @@ stages:
       - miniconda.sh
 
 
-.build_linux_template:
+build_linux:
   extends: .build_template
+  variables:
+    BUILD_EGG: "true"
   tags:
     - docker
   image: continuumio/conda-concourse-ci
   before_script:
     - rm -f /root/.condarc
     - rm -rf /root/.conda
+  script:
+    - python3 ./bob/devtools/bootstrap.py -vv build
+    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+    - conda activate base
+    - python3 ./bob/devtools/build.py -vv --twine-check
   artifacts:
     paths:
+      - dist/*.zip
+      - sphinx
       - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
       - ${CONDA_ROOT}/conda-bld/noarch/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
@@ -44,64 +48,23 @@ stages:
     key: "linux-cache"
 
 
-.build_macosx_template:
+build_macosx:
   extends: .build_template
   tags:
     - macosx
-  artifacts:
-    paths:
-      - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
-      - ${CONDA_ROOT}/conda-bld/noarch/*.conda
-      - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
-      - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
-  cache:
-    key: "macosx-cache"
-
-
-build_linux_36:
-  extends: .build_linux_template
-  variables:
-    PYTHON_VERSION: "3.6"
-
-build_linux_37:
-  extends: .build_linux_template
-  variables:
-    PYTHON_VERSION: "3.7"
-
-build_linux_38:
-  extends: .build_linux_template
-  variables:
-    PYTHON_VERSION: "3.8"
-    BUILD_EGG: "true"
   script:
     - python3 ./bob/devtools/bootstrap.py -vv build
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
     - conda activate base
-    - python3 ./bob/devtools/build.py -vv --twine-check
+    - python3 ./bob/devtools/build.py -vv
   artifacts:
     paths:
-      - dist/*.zip
-      - sphinx
-      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
+      - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
       - ${CONDA_ROOT}/conda-bld/noarch/*.conda
-      - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
+      - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
       - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
-
-
-build_macosx_36:
-  extends: .build_macosx_template
-  variables:
-    PYTHON_VERSION: "3.6"
-
-build_macosx_37:
-  extends: .build_macosx_template
-  variables:
-    PYTHON_VERSION: "3.7"
-
-build_macosx_38:
-  extends: .build_macosx_template
-  variables:
-    PYTHON_VERSION: "3.8"
+  cache:
+    key: "macosx-cache"
 
 
 # Deploy targets
@@ -118,12 +81,8 @@ build_macosx_38:
     - bdt ci deploy -vv
     - bdt ci clean -vv
   dependencies:
-    - build_linux_36
-    - build_linux_37
-    - build_linux_38
-    - build_macosx_36
-    - build_macosx_37
-    - build_macosx_38
+    - build_linux
+    - build_macosx
   tags:
     - docker
   cache:
@@ -164,12 +123,8 @@ pypi:
     - bdt ci pypi -vv dist/*.zip
     - bdt ci clean -vv
   dependencies:
-    - build_linux_36
-    - build_linux_37
-    - build_linux_38
-    - build_macosx_36
-    - build_macosx_37
-    - build_macosx_38
+    - build_linux
+    - build_macosx
   tags:
     - docker
   cache:

bob/devtools/build.py

@@ -102,7 +102,9 @@ def next_build_number(channel_url, basename):
     remove_conda_loggers()
 
     # get the channel index
-    channel_urls = calculate_channel_urls([channel_url], prepend=False, use_local=False)
+    channel_urls = calculate_channel_urls(
+        [channel_url], prepend=False, use_local=False
+    )
     logger.debug("Downloading channel index from %s", channel_urls)
     index = fetch_index(channel_urls=channel_urls)
 
@@ -215,7 +217,7 @@ def make_conda_config(config, python, append_file, condarc_options):
 def get_output_path(metadata, config):
     """Renders the recipe and returns the name of the output file."""
 
-    return conda_build.api.get_output_file_paths(metadata, config=config)[0]
+    return conda_build.api.get_output_file_paths(metadata, config=config)
 
 
 def get_rendered_metadata(recipe_dir, config):
@@ -565,7 +567,6 @@ def base_build(
     group,
     recipe_dir,
     conda_build_config,
-    python_version,
     condarc_options,
 ):
     """Builds a non-beat/non-bob software dependence that doesn't exist on
@@ -590,11 +591,6 @@ def base_build(
         our internal webserver.  Currently, only "bob" or "beat" will work.
       recipe_dir: The directory containing the recipe's ``meta.yaml`` file
       conda_build_config: Path to the ``conda_build_config.yaml`` file to use
-      python_version: String with the python version to build for, in the format
-        ``x.y`` (should be passed even if not building a python package).  It
-        can also be set to ``noarch``, or ``None``.  If set to ``None``, then we
-        don't assume there is a python-specific version being built.  If set to
-        ``noarch``, then it is a python package without a specific build.
       condarc_options: Pre-parsed condarc options loaded from the respective YAML
         file
 
@@ -607,8 +603,7 @@ def base_build(
 
     # if you get to this point, tries to build the package
     channels = bootstrap.get_channels(
-        public=True, stable=True, server=server, intranet=intranet,
-        group=group
+        public=True, stable=True, server=server, intranet=intranet, group=group
     )
 
     if "channels" not in condarc_options:
@@ -619,54 +614,40 @@ def base_build(
         "\n  - ".join(condarc_options["channels"]),
     )
     logger.info("Merging conda configuration files...")
-    if python_version not in ("noarch", None):
-        conda_config = make_conda_config(
-            conda_build_config, python_version, None, condarc_options
-        )
-    else:
-        conda_config = make_conda_config(
-            conda_build_config, None, None, condarc_options
-        )
+    conda_config = make_conda_config(
+        conda_build_config, None, None, condarc_options
+    )
 
     metadata = get_rendered_metadata(recipe_dir, conda_config)
-
-    # handles different cases as explained on the description of
-    # ``python_version``
-    py_ver = python_version.replace(".", "") if python_version else None
-    if py_ver == "noarch":
-        py_ver = ""
     arch = conda_arch()
 
     # checks we should actually build this recipe
     if should_skip_build(metadata):
-        if py_ver is None:
-            logger.warn(
-                'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch
-            )
-        elif not py_ver:
-            logger.warn(
-                'Skipping UNSUPPORTED build of "%s" for (noarch) python '
-                "on %s",
-                recipe_dir,
-                arch,
-            )
-        else:
-            logger.warn(
-                'Skipping UNSUPPORTED build of "%s" for python-%s ' "on %s",
-                recipe_dir,
-                python_version,
-                arch,
-            )
+        logger.warn(
+            'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch
+        )
         return
 
-    path = get_output_path(metadata, conda_config)
+    paths = get_output_path(metadata, conda_config)
+    urls = [exists_on_channel(channels[0], os.path.basename(k)) for k in paths]
 
-    url = exists_on_channel(channels[0], os.path.basename(path))
-    if url is not None:
-        logger.info("Skipping build for %s as it exists (at %s)", path, url)
+    if all(urls):
+        logger.info(
+            "Skipping build(s) for recipe at '%s' as packages with matching "
+            "characteristics exist (%s)",
+            recipe_dir,
+            ", ".join(urls),
+        )
         return
 
-    # if you get to this point, just builds the package
+    if any(urls):
+        raise RuntimeError(
+            "One or more packages for recipe at '%s' already exist (%s). "
+            "Change the package build number to trigger a build." % \
+            (recipe_dir, ", ".join(urls)),
+        )
+
+    # if you get to this point, just builds the package(s)
     logger.info("Building %s", path)
     return conda_build.api.build(recipe_dir, config=conda_config)
 
@@ -718,14 +699,6 @@ if __name__ == "__main__":
         default=os.environ.get("CI_PROJECT_DIR", os.path.realpath(os.curdir)),
         help="The directory where the repo was cloned [default: %(default)s]",
     )
-    parser.add_argument(
-        "-p",
-        "--python-version",
-        default=os.environ.get(
-            "PYTHON_VERSION", "%d.%d" % sys.version_info[:2]
-        ),
-        help="The version of python to build for [default: %(default)s]",
-    )
     parser.add_argument(
         "-T",
         "--twine-check",
@@ -783,8 +756,9 @@ if __name__ == "__main__":
     bootstrap.set_environment("BOB_PACKAGE_VERSION", version)
 
     # create the build configuration
-    conda_build_config = os.path.join(mydir, "data", "conda_build_config.yaml")
-    recipe_append = os.path.join(mydir, "data", "recipe_append.yaml")
+    conda_build_config = os.path.join(args.work_dir, "conda",
+            "conda_build_config.yaml")
+    recipe_append = os.path.join(args.work_dir, "data", "recipe_append.yaml")
 
     condarc = os.path.join(args.conda_root, "condarc")
     logger.info("Loading (this build's) CONDARC file from %s...", condarc)
@@ -812,11 +786,10 @@ if __name__ == "__main__":
             args.group,
             recipe,
             conda_build_config,
-            args.python_version,
             condarc_options,
         )
 
-    public = (args.visibility == "public")
+    public = args.visibility == "public"
     channels = bootstrap.get_channels(
         public=public,
         stable=(not is_prerelease),
@@ -834,36 +807,34 @@ if __name__ == "__main__":
     )
     logger.info("Merging conda configuration files...")
     conda_config = make_conda_config(
-        conda_build_config, args.python_version, recipe_append, condarc_options
+        conda_build_config, None, recipe_append, condarc_options
     )
 
     recipe_dir = os.path.join(args.work_dir, "conda")
     metadata = get_rendered_metadata(recipe_dir, conda_config)
-    path = get_output_path(metadata, conda_config)
+    paths = get_output_path(metadata, conda_config)
 
     # asserts we're building at the right location
-    assert path.startswith(os.path.join(args.conda_root, "conda-bld")), (
-        'Output path for build (%s) does not start with "%s" - this '
-        "typically means this build is running on a shared builder and "
-        "the file ~/.conda/environments.txt is polluted with other "
-        "environment paths.  To fix, empty that file and set its mode "
-        "to read-only for all."
-        % (path, os.path.join(args.conda_root, "conda-bld"))
-    )
+    for path in paths:
+        assert path.startswith(os.path.join(args.conda_root, "conda-bld")), (
+            'Output path for build (%s) does not start with "%s" - this '
+            "typically means this build is running on a shared builder and "
+            "the file ~/.conda/environments.txt is polluted with other "
+            "environment paths.  To fix, empty that file and set its mode "
+            "to read-only for all."
+            % (path, os.path.join(args.conda_root, "conda-bld"))
+        )
+
+    # retrieve the current build number(s) for this build
+    build_numbers = [
+        next_build_number(channels[0], os.path.basename(k))[0] for k in paths
+    ]
 
-    # retrieve the current build number for this build
-    build_number, _ = next_build_number(channels[0], os.path.basename(path))
+    # homogenize to the largest build number
+    build_number = max([int(k) for k in build_numbers])
 
     # runs the build using the conda-build API
     arch = conda_arch()
-    logger.info(
-        "Building %s-%s-py%s (build: %d) for %s",
-        args.name,
-        version,
-        args.python_version.replace(".", ""),
-        build_number,
-        arch,
-    )
 
     # notice we cannot build from the pre-parsed metadata because it has already
     # resolved the "wrong" build number.  We'll have to reparse after setting the

bob/devtools/graph.py

@@ -124,7 +124,7 @@ def compute_adjencence_matrix(
         # pre-renders the recipe - figures out the destination
         metadata = get_rendered_metadata(recipe_dir, conda_config)
         rendered_recipe = get_parsed_recipe(metadata)
-        path = get_output_path(metadata, conda_config)
+        path = get_output_path(metadata, conda_config)[0]
 
         # gets the next build number
         build_number, _ = next_build_number(

bob/devtools/scripts/build.py

@@ -266,7 +266,7 @@ def build(
             continue
 
         rendered_recipe = get_parsed_recipe(metadata)
-        path = get_output_path(metadata, conda_config)
+        path = get_output_path(metadata, conda_config)[0]
 
         # gets the next build number
         build_number, _ = next_build_number(channels[0], os.path.basename(path))

bob/devtools/scripts/local.py

@@ -176,13 +176,6 @@ Examples:
     "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
     "printing to help you understand what will be done",
 )
-@click.option(
-    "-p",
-    "--python",
-    multiple=True,
-    help='Versions of python in the format "x.y" we should build for.  Pass '
-    "various times this option to build for multiple python versions",
-)
 @click.option(
     "-g",
     "--group",
@@ -196,7 +189,4 @@ Examples:
 def base_build(ctx, order, dry_run, python, group):
     """Run the CI build step locally."""
     set_up_environment_variables(python=python, name_space=group)
-
-    ctx.invoke(
-        ci.base_build, order=order, dry_run=dry_run, group=group, python=python
-    )
+    ctx.invoke(ci.base_build, order=order, dry_run=dry_run, group=group)

bob/devtools/scripts/rebuild.py

@@ -253,7 +253,7 @@ def rebuild(
             continue
 
         rendered_recipe = get_parsed_recipe(metadata)
-        path = get_output_path(metadata, conda_config)
+        path = get_output_path(metadata, conda_config)[0]
 
         # Get the latest build number
         build_number, existing = next_build_number(

conda/conda_build_config.yaml

+macos_min_version:
+  - 10.9
+macos_machine:
+  - x86_64-apple-darwin13.4.0
+MACOSX_DEPLOYMENT_TARGET:
+  - 10.9
+CONDA_BUILD_SYSROOT:            # [osx]
+  - /opt/MacOSX10.9.sdk         # [osx]
+# This helps CMAKE find the sysroot. See
+# https://cmake.org/cmake/help/v3.11/variable/CMAKE_OSX_SYSROOT.html
+SDKROOT:                        # [osx]
+  - /opt/MacOSX10.9.sdk         # [osx]
+# makes autotools verbose
+VERBOSE_AT:
+  - V=1
+# makes cmake verbose
+VERBOSE_CM:
+  - VERBOSE=1
+
+## the dependencies that we build against multiple versions
+python:
+  - 3.6
+  - 3.7
+  - 3.8

conda/meta.yaml

@@ -12,10 +12,10 @@ build:
     - {{ pin_subpackage(name) }}
   script:
     - cd {{ environ.get('RECIPE_DIR') + '/..' }}
-    {% if environ.get('BUILD_EGG') %}
+    {% if environ.get('BUILD_EGG') and not os.path.exists('dist') %}
     - python setup.py sdist --formats=zip
     {% endif %}
-    - python setup.py install --single-version-externally-managed --record record.txt
+    - {{ PYTHON }} -m pip install --no-deps --ignore-installed .
     # installs the documentation source, readme to share/doc so it is available
     # during test time
     - install -d "${PREFIX}/share/doc/{{ name }}"
@@ -24,7 +24,7 @@ build:
 requirements:
   host:
     - python {{ python }}
-    - setuptools {{ setuptools }}
+    - pip
   run:
     - python
     - setuptools
@@ -108,7 +108,9 @@ test:
     - bdt gitlab graph --help
     - bdt gitlab badges --help
     - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
+    {% if not os.path.exists('sphinx') %}
     - if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
+    {% endif %}
 
 about:
   home: https://www.idiap.ch/software/bob/

conda/recipe_append.yaml

+build:
+  script_env:
+    - DOCSERVER
+    - NOSE_EVAL_ATTR