Merge branch 'conda-pkg-support' into 'master'

Prepare bob.devtools for .conda package support (c.f. issue #41)

Closes #41

See merge request !122

.gitlab-ci.yml

@@ -33,6 +33,8 @@ stages:
   image: continuumio/conda-concourse-ci
   artifacts:
     paths:
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
+      - ${CONDA_ROOT}/conda-bld/noarch/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
       - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
   cache:
@@ -45,6 +47,8 @@ stages:
     - macosx
   artifacts:
     paths:
+      - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
+      - ${CONDA_ROOT}/conda-bld/noarch/*.conda
       - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
       - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
   cache:
@@ -70,6 +74,8 @@ build_linux_37:
     paths:
       - dist/*.zip
       - sphinx
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
+      - ${CONDA_ROOT}/conda-bld/noarch/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
       - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
 

bob/devtools/bootstrap.py

@@ -19,6 +19,8 @@ remote_max_retries: 50 #!final
 remote_read_timeout_secs: 180.0 #!final
 channels:
   - defaults
+conda_build: #!final
+  pkg_format: '2'
 """
 
 _SERVER = "http://www.idiap.ch"
@@ -216,6 +218,8 @@ def merge_conda_cache(cache, prefix, name):
     # move packages on cache/pkgs to pkgs_dir
     cached_pkgs_dir = os.path.join(cache, "pkgs")
     cached_packages = glob.glob(os.path.join(cached_pkgs_dir, "*.tar.bz2"))
+    cached_packages.extend(glob.glob(os.path.join(cached_pkgs_dir, "*.conda")))
+
     cached_packages = [
         k for k in cached_packages if not k.startswith(name + "-")
     ]

bob/devtools/build.py

@@ -65,8 +65,8 @@ def should_skip_build(metadata_tuples):
 def next_build_number(channel_url, basename):
     """Calculates the next build number of a package given the channel.
 
-    This function returns the next build number (integer) for a package given its
-    resulting tarball base filename (can be obtained with
+    This function returns the next build number (integer) for a package given
+    its resulting tarball base filename (can be obtained with
     :py:func:`get_output_path`).
 
 
@@ -88,8 +88,14 @@ def next_build_number(channel_url, basename):
     logger.debug("Downloading channel index from %s", channel_url)
     index = get_index(channel_urls=[channel_url], prepend=False)
 
-    # remove .tar.bz2 from name, then split from the end twice, on '-'
-    name, version, build = basename[:-8].rsplit("-", 2)
+    # remove .tar.bz2/.conda from name, then split from the end twice, on '-'
+    if basename.endswith('.tar.bz2'):
+        name, version, build = basename[:-8].rsplit("-", 2)
+    elif basename.endswith('.conda'):
+        name, version, build = basename[:-6].rsplit("-", 2)
+    else:
+        raise RuntimeError("Package name %s does not end in either " \
+                ".tar.bz2 or .conda" % (basename,))
 
     # remove the build number as we're looking for the next value
     # examples to be coped with:
@@ -206,7 +212,8 @@ def get_parsed_recipe(metadata):
 def exists_on_channel(channel_url, basename):
     """Checks on the given channel if a package with the specs exist.
 
-    This procedure always ignores the package hash code, if one is set
+    This procedure always ignores the package hash code, if one is set.  It
+    differentiates between `.conda` and `.tar.bz2` packages.
 
     Args:
 
@@ -214,16 +221,23 @@ def exists_on_channel(channel_url, basename):
         channel)
       basename: The basename of the tarball to search for
 
-    Returns: A complete package url, if the package already exists in the channel
-    or ``None`` otherwise.
+    Returns: A complete package url, if the package already exists in the
+    channel or ``None`` otherwise.
     """
 
     build_number, urls = next_build_number(channel_url, basename)
 
     def _get_build_number(name):
 
-        # remove .tar.bz2 from name, then split from the end twice, on '-'
-        name, version, build = name[:-8].rsplit("-", 2)
+        # remove .tar.bz2/.conda from name, then split from the end twice, on
+        # '-'
+        if name.endswith('.conda'):
+            name, version, build = name[:-6].rsplit("-", 2)
+        elif name.endswith('.tar.bz2'):
+            name, version, build = name[:-8].rsplit("-", 2)
+        else:
+            raise RuntimeError("Package name %s does not end in either " \
+                    ".tar.bz2 or .conda" % (name,))
 
         # remove the build number as we're looking for the next value
         # examples to be coped with:
@@ -239,9 +253,10 @@ def exists_on_channel(channel_url, basename):
     other_build_numbers = [_get_build_number(os.path.basename(k)) for k in urls]
 
     if self_build_number in other_build_numbers:
-        return "".join(
-            (channel_url, urls[other_build_numbers.index(self_build_number)])
-        )
+        candidate = urls[other_build_numbers.index(self_build_number)]
+        pkg_type = '.conda' if basename.endswith('.conda') else '.tar.bz2'
+        if candidate.endswith(pkg_type):  #match
+            return "".join(channel_url, candidate)
 
 
 def remove_pins(deps):

bob/devtools/data/gitlab-ci/docs.yaml

@@ -43,6 +43,7 @@ build:
     expire_in: 1 week
     paths:
       - sphinx
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
   tags:
     - docker

bob/devtools/data/gitlab-ci/single-package.yaml

@@ -46,6 +46,7 @@ stages:
   image: continuumio/conda-concourse-ci
   artifacts:
     paths:
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
 
 
@@ -55,6 +56,7 @@ stages:
     - macosx
   artifacts:
     paths:
+      - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
       - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
 
 
@@ -92,6 +94,7 @@ build_linux_37:
     paths:
       - dist/*.zip
       - sphinx
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
   cache:
     key: "build-py37"

bob/devtools/dav.py

@@ -121,10 +121,13 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True, includes=None):
 
         if f.startswith("."):
             continue
-        if not f.endswith(".tar.bz2"):
-            continue
 
-        name, version, build_string = f[:-8].rsplit("-", 2)
+        if f.endswith(".tar.bz2"):
+            name, version, build_string = f[:-8].rsplit("-", 2)
+        elif f.endswith(".conda"):
+            name, version, build_string = f[:-6].rsplit("-", 2)
+        else:
+            continue
 
         # see if this package should be included or not in our clean-up
         if (includes is not None) and (not includes.match(name)):

bob/devtools/scripts/ci.py

@@ -80,10 +80,10 @@ def base_deploy(dry_run):
     # afterwards)
     for arch in ("linux-64", "osx-64", "noarch"):
         # finds conda dependencies and uploads what we can find
-        package_path = os.path.join(
-            os.environ["CONDA_ROOT"], "conda-bld", arch, "*.tar.bz2"
-        )
-        deploy_packages = glob.glob(package_path)
+        base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
+        conda_paths = os.path.join(base_path, "*.conda")
+        tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
+        deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
 
         for k in deploy_packages:
 
@@ -167,10 +167,11 @@ def deploy(latest, dry_run):
     # afterwards)
     for arch in ("linux-64", "osx-64", "noarch"):
         # finds conda packages and uploads what we can find
-        package_path = os.path.join(
-            os.environ["CONDA_ROOT"], "conda-bld", arch, name + "*.tar.bz2"
-        )
-        deploy_packages = glob.glob(package_path)
+        base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
+        conda_paths = os.path.join(base_path, "*.conda")
+        tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
+        deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
+
         for k in deploy_packages:
             deploy_conda_package(
                 k,
@@ -504,16 +505,13 @@ def test(ctx, dry_run):
 
     from .test import test
 
+    base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", "*",
+            os.environ["CI_PROJECT_NAME"])
+
     ctx.invoke(
         test,
-        package=glob.glob(
-            os.path.join(
-                os.environ["CONDA_ROOT"],
-                "conda-bld",
-                "*",
-                os.environ["CI_PROJECT_NAME"] + "*.tar.bz2",
-            )
-        ),
+        package=glob.glob(base_path + "*.conda") + \
+                glob.glob(base_path + "*.tar.bz2"),
         condarc=condarc,
         config=variants_file,
         append_file=append_file,

bob/devtools/scripts/test.py

@@ -37,13 +37,13 @@ Examples:
   1. Tests conda package:
 
 \b
-     $ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2
+     $ bdt test -vv /path/to/conda-package-v1.0.0.conda
 
 
   2. Tests multiple conda packages, one after the other:
 
 \b
-     $ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2 /path/to/other-conda-package-v2.0.0.tar.bz2
+     $ bdt test -vv /path/to/conda-package-v1.0.0.conda /path/to/other-conda-package-v2.0.0.conda
 
 """
 )

deps/python-gitlab/meta.yaml

 {% set name = "python-gitlab" %}
-{% set version = "1.10.0" %}
+{% set version = "1.12.1" %}
 
 package:
   name: {{ name|lower }}
@@ -7,7 +7,7 @@ package:
 
 source:
   url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
-  sha256: c4e91b9367c54c3049d702c35dd966f9e5a8989dae1be56ef7a1c35c2b235a58
+  sha256: 984e110c1f76fd939652c30ce3101267a7064e34417cbfc4687e6106d4db54ec
 
 build:
   number: 0