Commit 862fc0f9 authored by André Anjos's avatar André Anjos 💬

Merge branch 'conda-pkg-support' into 'master'

Prepare bob.devtools for .conda package support (c.f. issue #41)

Closes #41

See merge request !122
parents 1603b8f9 25b518e4
Pipeline #35256 passed with stages
in 8 minutes and 9 seconds
......@@ -33,6 +33,8 @@ stages:
image: continuumio/conda-concourse-ci
artifacts:
paths:
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
cache:
......@@ -45,6 +47,8 @@ stages:
- macosx
artifacts:
paths:
- ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
cache:
......@@ -70,6 +74,8 @@ build_linux_37:
paths:
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
......
......@@ -19,6 +19,8 @@ remote_max_retries: 50 #!final
remote_read_timeout_secs: 180.0 #!final
channels:
- defaults
conda_build: #!final
pkg_format: '2'
"""
_SERVER = "http://www.idiap.ch"
......@@ -216,6 +218,8 @@ def merge_conda_cache(cache, prefix, name):
# move packages on cache/pkgs to pkgs_dir
cached_pkgs_dir = os.path.join(cache, "pkgs")
cached_packages = glob.glob(os.path.join(cached_pkgs_dir, "*.tar.bz2"))
cached_packages.extend(glob.glob(os.path.join(cached_pkgs_dir, "*.conda")))
cached_packages = [
k for k in cached_packages if not k.startswith(name + "-")
]
......
......@@ -65,8 +65,8 @@ def should_skip_build(metadata_tuples):
def next_build_number(channel_url, basename):
"""Calculates the next build number of a package given the channel.
This function returns the next build number (integer) for a package given its
resulting tarball base filename (can be obtained with
This function returns the next build number (integer) for a package given
its resulting tarball base filename (can be obtained with
:py:func:`get_output_path`).
......@@ -88,8 +88,14 @@ def next_build_number(channel_url, basename):
logger.debug("Downloading channel index from %s", channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
# remove .tar.bz2 from name, then split from the end twice, on '-'
name, version, build = basename[:-8].rsplit("-", 2)
# remove .tar.bz2/.conda from name, then split from the end twice, on '-'
if basename.endswith('.tar.bz2'):
name, version, build = basename[:-8].rsplit("-", 2)
elif basename.endswith('.conda'):
name, version, build = basename[:-6].rsplit("-", 2)
else:
raise RuntimeError("Package name %s does not end in either " \
".tar.bz2 or .conda" % (basename,))
# remove the build number as we're looking for the next value
# examples to be coped with:
......@@ -206,7 +212,8 @@ def get_parsed_recipe(metadata):
def exists_on_channel(channel_url, basename):
"""Checks on the given channel if a package with the specs exist.
This procedure always ignores the package hash code, if one is set
This procedure always ignores the package hash code, if one is set. It
differentiates between `.conda` and `.tar.bz2` packages.
Args:
......@@ -214,16 +221,23 @@ def exists_on_channel(channel_url, basename):
channel)
basename: The basename of the tarball to search for
Returns: A complete package url, if the package already exists in the channel
or ``None`` otherwise.
Returns: A complete package url, if the package already exists in the
channel or ``None`` otherwise.
"""
build_number, urls = next_build_number(channel_url, basename)
def _get_build_number(name):
# remove .tar.bz2 from name, then split from the end twice, on '-'
name, version, build = name[:-8].rsplit("-", 2)
# remove .tar.bz2/.conda from name, then split from the end twice, on
# '-'
if name.endswith('.conda'):
name, version, build = name[:-6].rsplit("-", 2)
elif name.endswith('.tar.bz2'):
name, version, build = name[:-8].rsplit("-", 2)
else:
raise RuntimeError("Package name %s does not end in either " \
".tar.bz2 or .conda" % (name,))
# remove the build number as we're looking for the next value
# examples to be coped with:
......@@ -239,9 +253,10 @@ def exists_on_channel(channel_url, basename):
other_build_numbers = [_get_build_number(os.path.basename(k)) for k in urls]
if self_build_number in other_build_numbers:
return "".join(
(channel_url, urls[other_build_numbers.index(self_build_number)])
)
candidate = urls[other_build_numbers.index(self_build_number)]
pkg_type = '.conda' if basename.endswith('.conda') else '.tar.bz2'
if candidate.endswith(pkg_type): #match
return "".join(channel_url, candidate)
def remove_pins(deps):
......
......@@ -43,6 +43,7 @@ build:
expire_in: 1 week
paths:
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
tags:
- docker
......
......@@ -46,6 +46,7 @@ stages:
image: continuumio/conda-concourse-ci
artifacts:
paths:
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
......@@ -55,6 +56,7 @@ stages:
- macosx
artifacts:
paths:
- ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
......@@ -92,6 +94,7 @@ build_linux_37:
paths:
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
key: "build-py37"
......
......@@ -121,10 +121,13 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True, includes=None):
if f.startswith("."):
continue
if not f.endswith(".tar.bz2"):
continue
name, version, build_string = f[:-8].rsplit("-", 2)
if f.endswith(".tar.bz2"):
name, version, build_string = f[:-8].rsplit("-", 2)
elif f.endswith(".conda"):
name, version, build_string = f[:-6].rsplit("-", 2)
else:
continue
# see if this package should be included or not in our clean-up
if (includes is not None) and (not includes.match(name)):
......
......@@ -80,10 +80,10 @@ def base_deploy(dry_run):
# afterwards)
for arch in ("linux-64", "osx-64", "noarch"):
# finds conda dependencies and uploads what we can find
package_path = os.path.join(
os.environ["CONDA_ROOT"], "conda-bld", arch, "*.tar.bz2"
)
deploy_packages = glob.glob(package_path)
base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
conda_paths = os.path.join(base_path, "*.conda")
tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
for k in deploy_packages:
......@@ -167,10 +167,11 @@ def deploy(latest, dry_run):
# afterwards)
for arch in ("linux-64", "osx-64", "noarch"):
# finds conda packages and uploads what we can find
package_path = os.path.join(
os.environ["CONDA_ROOT"], "conda-bld", arch, name + "*.tar.bz2"
)
deploy_packages = glob.glob(package_path)
base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
conda_paths = os.path.join(base_path, "*.conda")
tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
for k in deploy_packages:
deploy_conda_package(
k,
......@@ -504,16 +505,13 @@ def test(ctx, dry_run):
from .test import test
base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", "*",
os.environ["CI_PROJECT_NAME"])
ctx.invoke(
test,
package=glob.glob(
os.path.join(
os.environ["CONDA_ROOT"],
"conda-bld",
"*",
os.environ["CI_PROJECT_NAME"] + "*.tar.bz2",
)
),
package=glob.glob(base_path + "*.conda") + \
glob.glob(base_path + "*.tar.bz2"),
condarc=condarc,
config=variants_file,
append_file=append_file,
......
......@@ -37,13 +37,13 @@ Examples:
1. Tests conda package:
\b
$ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2
$ bdt test -vv /path/to/conda-package-v1.0.0.conda
2. Tests multiple conda packages, one after the other:
\b
$ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2 /path/to/other-conda-package-v2.0.0.tar.bz2
$ bdt test -vv /path/to/conda-package-v1.0.0.conda /path/to/other-conda-package-v2.0.0.conda
"""
)
......
{% set name = "python-gitlab" %}
{% set version = "1.10.0" %}
{% set version = "1.12.1" %}
package:
name: {{ name|lower }}
......@@ -7,7 +7,7 @@ package:
source:
url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
sha256: c4e91b9367c54c3049d702c35dd966f9e5a8989dae1be56ef7a1c35c2b235a58
sha256: 984e110c1f76fd939652c30ce3101267a7064e34417cbfc4687e6106d4db54ec
build:
number: 0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment