.gitlab-ci.yml

@@ -33,6 +33,8 @@ stages:
   image: continuumio/conda-concourse-ci
   artifacts:
     paths:
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
+      - ${CONDA_ROOT}/conda-bld/noarch/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
       - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
   cache:
@@ -45,6 +47,8 @@ stages:
     - macosx
   artifacts:
     paths:
+      - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
+      - ${CONDA_ROOT}/conda-bld/noarch/*.conda
       - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
       - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
   cache:
@@ -70,6 +74,8 @@ build_linux_37:
     paths:
       - dist/*.zip
       - sphinx
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
+      - ${CONDA_ROOT}/conda-bld/noarch/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
       - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
 

bob/devtools/bootstrap.py

@@ -19,6 +19,8 @@ remote_max_retries: 50 #!final
 remote_read_timeout_secs: 180.0 #!final
 channels:
   - defaults
+conda_build: #!final
+  pkg_format: '2'
 """
 
 _SERVER = "http://www.idiap.ch"
@@ -50,8 +52,8 @@ def do_hack(project_dir):
     """
     This function is supposed to be for temporary usage.
 
-    It implements hacks for the issues: https://gitlab.idiap.ch/bob/bob.devtools/merge_requests/112
-    and https://github.com/conda/conda-build/issues/3767)
+    It implements hacks for the issue:
+    https://gitlab.idiap.ch/bob/bob.devtools/merge_requests/112
 
     """
 
@@ -60,47 +62,13 @@ def do_hack(project_dir):
 
     git_ignore_file = os.path.join(project_dir, ".gitignore")
     if os.path.exists(git_ignore_file):
-        logger.warning('Removing ".gitignore" to overcome issues with ripgrep')
         logger.warning(
-            "See https://gitlab.idiap.ch/bob/bob.devtools/merge_requests/112"
+            "Removing .gitignore to avoid issue with ripgrep (see "
+            "https://gitlab.idiap.ch/bob/bob.devtools/merge_requests/112)"
         )
         os.unlink(git_ignore_file)
     #### END OF HACK
 
-    #### HACK that avoids this issue: https://github.com/conda/conda-build/issues/3767
-    license_file = os.path.join(project_dir, "LICENSE")
-    if not os.path.exists(license_file):
-        license_file = os.path.join(project_dir, "LICENSE.AGPL")
-
-    recipe_dir = os.path.join(project_dir, "conda")
-    if os.path.exists(license_file) and os.path.exists(recipe_dir):
-        logger.warning(
-            "Copying LICENSE file to `./conda` dir to avoid issue with conda build (https://github.com/conda/conda-build/issues/3767)"
-        )
-        logger.warning(
-            "Replacing ../LICENSE to LICENSE (https://github.com/conda/conda-build/issues/3767)"
-        )
-        shutil.copyfile(
-            license_file,
-            os.path.join(recipe_dir, os.path.basename(license_file)),
-        )
-
-        # Checking COPYING file just in case
-        copying_file = os.path.join(project_dir, "COPYING")
-        if os.path.exists(copying_file):
-            shutil.copyfile(copying_file, os.path.join(recipe_dir, "COPYING"))
-
-        meta_file = os.path.join(recipe_dir, "meta.yaml")
-        recipe = open(meta_file).readlines()
-        recipe = [
-            l.replace("../COPYING", "COPYING")
-            .replace("../LICENSE", "LICENSE")
-            .replace("../LICENSE.AGPL", "LICENSE.AGPL")
-            for l in recipe
-        ]
-        open(meta_file, "wt").write("".join(recipe))
-    #### END OF HACK
-
 
 def set_environment(name, value, env=os.environ):
     """Function to setup the environment variable and print debug message.
@@ -216,6 +184,8 @@ def merge_conda_cache(cache, prefix, name):
     # move packages on cache/pkgs to pkgs_dir
     cached_pkgs_dir = os.path.join(cache, "pkgs")
     cached_packages = glob.glob(os.path.join(cached_pkgs_dir, "*.tar.bz2"))
+    cached_packages.extend(glob.glob(os.path.join(cached_pkgs_dir, "*.conda")))
+
     cached_packages = [
         k for k in cached_packages if not k.startswith(name + "-")
     ]

bob/devtools/build.py

@@ -65,8 +65,8 @@ def should_skip_build(metadata_tuples):
 def next_build_number(channel_url, basename):
     """Calculates the next build number of a package given the channel.
 
-    This function returns the next build number (integer) for a package given its
-    resulting tarball base filename (can be obtained with
+    This function returns the next build number (integer) for a package given
+    its resulting tarball base filename (can be obtained with
     :py:func:`get_output_path`).
 
 
@@ -88,8 +88,14 @@ def next_build_number(channel_url, basename):
     logger.debug("Downloading channel index from %s", channel_url)
     index = get_index(channel_urls=[channel_url], prepend=False)
 
-    # remove .tar.bz2 from name, then split from the end twice, on '-'
-    name, version, build = basename[:-8].rsplit("-", 2)
+    # remove .tar.bz2/.conda from name, then split from the end twice, on '-'
+    if basename.endswith('.tar.bz2'):
+        name, version, build = basename[:-8].rsplit("-", 2)
+    elif basename.endswith('.conda'):
+        name, version, build = basename[:-6].rsplit("-", 2)
+    else:
+        raise RuntimeError("Package name %s does not end in either " \
+                ".tar.bz2 or .conda" % (basename,))
 
     # remove the build number as we're looking for the next value
     # examples to be coped with:
@@ -206,7 +212,8 @@ def get_parsed_recipe(metadata):
 def exists_on_channel(channel_url, basename):
     """Checks on the given channel if a package with the specs exist.
 
-    This procedure always ignores the package hash code, if one is set
+    This procedure always ignores the package hash code, if one is set.  It
+    differentiates between `.conda` and `.tar.bz2` packages.
 
     Args:
 
@@ -214,34 +221,45 @@ def exists_on_channel(channel_url, basename):
         channel)
       basename: The basename of the tarball to search for
 
-    Returns: A complete package url, if the package already exists in the channel
-    or ``None`` otherwise.
+    Returns: A complete package url, if the package already exists in the
+    channel or ``None`` otherwise.
     """
 
     build_number, urls = next_build_number(channel_url, basename)
 
     def _get_build_number(name):
 
-        # remove .tar.bz2 from name, then split from the end twice, on '-'
-        name, version, build = name[:-8].rsplit("-", 2)
+        # remove .tar.bz2/.conda from name, then split from the end twice, on
+        # '-'
+        if name.endswith('.conda'):
+            name, version, build = name[:-6].rsplit("-", 2)
+        elif name.endswith('.tar.bz2'):
+            name, version, build = name[:-8].rsplit("-", 2)
+        else:
+            raise RuntimeError("Package name %s does not end in either " \
+                    ".tar.bz2 or .conda" % (name,))
 
         # remove the build number as we're looking for the next value
         # examples to be coped with:
         # vlfeat-0.9.20-0 -> '0'
         # vlfeat-0.9.21-h18fa195_0 -> 'h18fa195_0'
         # tqdm-4.11.1-py36_0 -> 'py36_0'
+        # untokenize-0.1.1-py_0.conda -> 'py_0'
         # websocket-client-0.47.0-py27haf68d3b_0 -> 'py27haf68d3b_0'
         # websocket-client-0.47.0-py36haf68d3b_0 -> 'py36haf68d3b_0'
         s = build.rsplit("_", 1)
         return s[1] if len(s) == 2 else s[0]
 
     self_build_number = _get_build_number(basename)
-    other_build_numbers = [_get_build_number(os.path.basename(k)) for k in urls]
+    other_build_numbers = dict(
+            [(k, _get_build_number(os.path.basename(k))) for k in urls]
+            )
 
-    if self_build_number in other_build_numbers:
-        return "".join(
-            (channel_url, urls[other_build_numbers.index(self_build_number)])
-        )
+    if self_build_number in other_build_numbers.values():
+        pkg_type = '.conda' if basename.endswith('.conda') else '.tar.bz2'
+        for k, v in other_build_numbers.items():
+            if k.endswith(pkg_type):  #match
+                return "".join((channel_url, k))
 
 
 def remove_pins(deps):
@@ -744,8 +762,8 @@ if __name__ == "__main__":
         condarc_options["croot"] = os.path.join(prefix, "conda-bld")
 
     # builds all dependencies in the 'deps' subdirectory - or at least checks
-    # these dependencies are already available; these dependencies go directly to
-    # the public channel once built
+    # these dependencies are already available; these dependencies go directly
+    # to the public channel once built
     recipes = load_order_file(os.path.join("deps", "order.txt"))
     for k, recipe in enumerate([os.path.join("deps", k) for k in recipes]):
 

bob/devtools/data/gitlab-ci/docs.yaml

@@ -43,6 +43,7 @@ build:
     expire_in: 1 week
     paths:
       - sphinx
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
   tags:
     - docker

bob/devtools/data/gitlab-ci/single-package.yaml

@@ -46,6 +46,7 @@ stages:
   image: continuumio/conda-concourse-ci
   artifacts:
     paths:
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
 
 
@@ -55,6 +56,7 @@ stages:
     - macosx
   artifacts:
     paths:
+      - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
       - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
 
 
@@ -92,6 +94,7 @@ build_linux_37:
     paths:
       - dist/*.zip
       - sphinx
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
       - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
   cache:
     key: "build-py37"

bob/devtools/dav.py

@@ -121,10 +121,13 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True, includes=None):
 
         if f.startswith("."):
             continue
-        if not f.endswith(".tar.bz2"):
-            continue
 
-        name, version, build_string = f[:-8].rsplit("-", 2)
+        if f.endswith(".tar.bz2"):
+            name, version, build_string = f[:-8].rsplit("-", 2)
+        elif f.endswith(".conda"):
+            name, version, build_string = f[:-6].rsplit("-", 2)
+        else:
+            continue
 
         # see if this package should be included or not in our clean-up
         if (includes is not None) and (not includes.match(name)):

bob/devtools/scripts/ci.py

@@ -80,10 +80,10 @@ def base_deploy(dry_run):
     # afterwards)
     for arch in ("linux-64", "osx-64", "noarch"):
         # finds conda dependencies and uploads what we can find
-        package_path = os.path.join(
-            os.environ["CONDA_ROOT"], "conda-bld", arch, "*.tar.bz2"
-        )
-        deploy_packages = glob.glob(package_path)
+        base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
+        conda_paths = os.path.join(base_path, "*.conda")
+        tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
+        deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
 
         for k in deploy_packages:
 
@@ -167,10 +167,11 @@ def deploy(latest, dry_run):
     # afterwards)
     for arch in ("linux-64", "osx-64", "noarch"):
         # finds conda packages and uploads what we can find
-        package_path = os.path.join(
-            os.environ["CONDA_ROOT"], "conda-bld", arch, name + "*.tar.bz2"
-        )
-        deploy_packages = glob.glob(package_path)
+        base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
+        conda_paths = os.path.join(base_path, "*.conda")
+        tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
+        deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
+
         for k in deploy_packages:
             deploy_conda_package(
                 k,
@@ -504,16 +505,13 @@ def test(ctx, dry_run):
 
     from .test import test
 
+    base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", "*",
+            os.environ["CI_PROJECT_NAME"])
+
     ctx.invoke(
         test,
-        package=glob.glob(
-            os.path.join(
-                os.environ["CONDA_ROOT"],
-                "conda-bld",
-                "*",
-                os.environ["CI_PROJECT_NAME"] + "*.tar.bz2",
-            )
-        ),
+        package=glob.glob(base_path + "*.conda") + \
+                glob.glob(base_path + "*.tar.bz2"),
         condarc=condarc,
         config=variants_file,
         append_file=append_file,

bob/devtools/scripts/test.py

@@ -37,13 +37,13 @@ Examples:
   1. Tests conda package:
 
 \b
-     $ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2
+     $ bdt test -vv /path/to/conda-package-v1.0.0.conda
 
 
   2. Tests multiple conda packages, one after the other:
 
 \b
-     $ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2 /path/to/other-conda-package-v2.0.0.tar.bz2
+     $ bdt test -vv /path/to/conda-package-v1.0.0.conda /path/to/other-conda-package-v2.0.0.conda
 
 """
 )

bob/devtools/templates/conda/meta.yaml

@@ -56,4 +56,3 @@ about:
   home: https://www.idiap.ch/software/(( group ))/
   license: (% if license == 'gplv3' %)GNU General Public License v3 (GPLv3)(% else %)BSD 3-Clause(% endif %)
   license_family: (% if license == 'gplv3' %)GPL(% else %)BSD(% endif %)
-  license_file: (% if license == 'gplv3' %)../COPYING(% else %)../LICENSE(% endif %)

conda/meta.yaml

@@ -103,7 +103,7 @@ test:
     - bdt dav clean-betas --help
     - bdt dav upload --help
     - bdt gitlab process-pipelines --help
-    - bdt gitlab get-pipelines --help    
+    - bdt gitlab get-pipelines --help
     - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
     - if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
 
@@ -112,4 +112,3 @@ about:
   license: BSD 3-Clause
   summary: Tools for development and CI integration of Bob packages
   license_family: BSD
-  license_file: ../LICENSE

deps/python-gitlab/meta.yaml

 {% set name = "python-gitlab" %}
-{% set version = "1.10.0" %}
+{% set version = "1.12.1" %}
 
 package:
   name: {{ name|lower }}
@@ -7,7 +7,7 @@ package:
 
 source:
   url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
-  sha256: c4e91b9367c54c3049d702c35dd966f9e5a8989dae1be56ef7a1c35c2b235a58
+  sha256: 984e110c1f76fd939652c30ce3101267a7064e34417cbfc4687e6106d4db54ec
 
 build:
   number: 0