From bf43f231e753ef0cc53848941e81f3797a8485c6 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Fri, 18 Oct 2019 16:30:29 +0200
Subject: [PATCH] [many] Prepare bob.devtools for .conda package support (c.f.
issue #41)
---
bob/devtools/bootstrap.py | 2 ++
bob/devtools/build.py | 25 ++++++++++++----
bob/devtools/data/gitlab-ci/docs.yaml | 1 +
.../data/gitlab-ci/single-package.yaml | 3 ++
bob/devtools/dav.py | 2 +-
bob/devtools/scripts/ci.py | 30 +++++++++----------
bob/devtools/scripts/test.py | 4 +--
7 files changed, 42 insertions(+), 25 deletions(-)
diff --git a/bob/devtools/bootstrap.py b/bob/devtools/bootstrap.py
index a79709c8..44c965c7 100644
--- a/bob/devtools/bootstrap.py
+++ b/bob/devtools/bootstrap.py
@@ -216,6 +216,8 @@ def merge_conda_cache(cache, prefix, name):
# move packages on cache/pkgs to pkgs_dir
cached_pkgs_dir = os.path.join(cache, "pkgs")
cached_packages = glob.glob(os.path.join(cached_pkgs_dir, "*.tar.bz2"))
+ cached_packages.extend(glob.glob(os.path.join(cached_pkgs_dir, "*.conda")))
+
cached_packages = [
k for k in cached_packages if not k.startswith(name + "-")
]
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index 81a17b07..5f3b673c 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -88,8 +88,14 @@ def next_build_number(channel_url, basename):
logger.debug("Downloading channel index from %s", channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
- # remove .tar.bz2 from name, then split from the end twice, on '-'
- name, version, build = basename[:-8].rsplit("-", 2)
+ # remove .tar.bz2/.conda from name, then split from the end twice, on '-'
+ if basename.endswith('.tar.bz2'):
+ name, version, build = basename[:-8].rsplit("-", 2)
+ elif basename.endswith('.conda'):
+ name, version, build = basename[:-6].rsplit("-", 2)
+ else:
+ raise RuntimeError("Package name %s does not end in either " \
+ ".tar.bz2 or .conda" % (basename,))
# remove the build number as we're looking for the next value
# examples to be coped with:
@@ -214,16 +220,23 @@ def exists_on_channel(channel_url, basename):
channel)
basename: The basename of the tarball to search for
- Returns: A complete package url, if the package already exists in the channel
- or ``None`` otherwise.
+ Returns: A complete package url, if the package already exists in the
+ channel or ``None`` otherwise.
"""
build_number, urls = next_build_number(channel_url, basename)
def _get_build_number(name):
- # remove .tar.bz2 from name, then split from the end twice, on '-'
- name, version, build = name[:-8].rsplit("-", 2)
+ # remove .tar.bz2/.conda from name, then split from the end twice, on
+ # '-'
+ if name.endswith('.tar.bz2'):
+ name, version, build = name[:-8].rsplit("-", 2)
+ elif name.endswith('.conda'):
+ name, version, build = name[:-6].rsplit("-", 2)
+ else:
+ raise RuntimeError("Package name %s does not end in either " \
+ ".tar.bz2 or .conda" % (name,))
# remove the build number as we're looking for the next value
# examples to be coped with:
diff --git a/bob/devtools/data/gitlab-ci/docs.yaml b/bob/devtools/data/gitlab-ci/docs.yaml
index 79f16f40..0406d6de 100644
--- a/bob/devtools/data/gitlab-ci/docs.yaml
+++ b/bob/devtools/data/gitlab-ci/docs.yaml
@@ -43,6 +43,7 @@ build:
expire_in: 1 week
paths:
- sphinx
+ - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
tags:
- docker
diff --git a/bob/devtools/data/gitlab-ci/single-package.yaml b/bob/devtools/data/gitlab-ci/single-package.yaml
index 03e8e72b..1af05699 100644
--- a/bob/devtools/data/gitlab-ci/single-package.yaml
+++ b/bob/devtools/data/gitlab-ci/single-package.yaml
@@ -46,6 +46,7 @@ stages:
image: continuumio/conda-concourse-ci
artifacts:
paths:
+ - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
@@ -55,6 +56,7 @@ stages:
- macosx
artifacts:
paths:
+ - ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
@@ -92,6 +94,7 @@ build_linux_37:
paths:
- dist/*.zip
- sphinx
+ - ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
key: "build-py37"
diff --git a/bob/devtools/dav.py b/bob/devtools/dav.py
index 6cc77dfd..d9fe441c 100644
--- a/bob/devtools/dav.py
+++ b/bob/devtools/dav.py
@@ -121,7 +121,7 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True, includes=None):
if f.startswith("."):
continue
- if not f.endswith(".tar.bz2"):
+ if not (f.endswith(".conda") or f.endswith(".tar.bz2")):
continue
name, version, build_string = f[:-8].rsplit("-", 2)
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index 850005ee..adf71eea 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -80,10 +80,10 @@ def base_deploy(dry_run):
# afterwards)
for arch in ("linux-64", "osx-64", "noarch"):
# finds conda dependencies and uploads what we can find
- package_path = os.path.join(
- os.environ["CONDA_ROOT"], "conda-bld", arch, "*.tar.bz2"
- )
- deploy_packages = glob.glob(package_path)
+ base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
+ conda_paths = os.path.join(base_path, "*.conda")
+ tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
+ deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
for k in deploy_packages:
@@ -167,10 +167,11 @@ def deploy(latest, dry_run):
# afterwards)
for arch in ("linux-64", "osx-64", "noarch"):
# finds conda packages and uploads what we can find
- package_path = os.path.join(
- os.environ["CONDA_ROOT"], "conda-bld", arch, name + "*.tar.bz2"
- )
- deploy_packages = glob.glob(package_path)
+ base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
+ conda_paths = os.path.join(base_path, "*.conda")
+ tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
+ deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
+
for k in deploy_packages:
deploy_conda_package(
k,
@@ -504,16 +505,13 @@ def test(ctx, dry_run):
from .test import test
+ base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", "*",
+ os.environ["CI_PROJECT_NAME"])
+
ctx.invoke(
test,
- package=glob.glob(
- os.path.join(
- os.environ["CONDA_ROOT"],
- "conda-bld",
- "*",
- os.environ["CI_PROJECT_NAME"] + "*.tar.bz2",
- )
- ),
+ package=glob.glob(base_path + "*.conda")) + \
+ glob.glob(base_path + "*.tar.bz2"))
condarc=condarc,
config=variants_file,
append_file=append_file,
diff --git a/bob/devtools/scripts/test.py b/bob/devtools/scripts/test.py
index 32650201..d1f28ae9 100644
--- a/bob/devtools/scripts/test.py
+++ b/bob/devtools/scripts/test.py
@@ -37,13 +37,13 @@ Examples:
1. Tests conda package:
\b
- $ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2
+ $ bdt test -vv /path/to/conda-package-v1.0.0.conda
2. Tests multiple conda packages, one after the other:
\b
- $ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2 /path/to/other-conda-package-v2.0.0.tar.bz2
+ $ bdt test -vv /path/to/conda-package-v1.0.0.conda /path/to/other-conda-package-v2.0.0.conda
"""
)
--
GitLab