Commit 862fc0f9 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'conda-pkg-support' into 'master'

Prepare bob.devtools for .conda package support (c.f. issue #41)

Closes #41

See merge request !122
parents 1603b8f9 25b518e4
Pipeline #35256 passed with stages
in 8 minutes and 9 seconds
...@@ -33,6 +33,8 @@ stages: ...@@ -33,6 +33,8 @@ stages:
image: continuumio/conda-concourse-ci image: continuumio/conda-concourse-ci
artifacts: artifacts:
paths: paths:
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
cache: cache:
...@@ -45,6 +47,8 @@ stages: ...@@ -45,6 +47,8 @@ stages:
- macosx - macosx
artifacts: artifacts:
paths: paths:
- ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
cache: cache:
...@@ -70,6 +74,8 @@ build_linux_37: ...@@ -70,6 +74,8 @@ build_linux_37:
paths: paths:
- dist/*.zip - dist/*.zip
- sphinx - sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/noarch/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
- ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/noarch/*.tar.bz2
......
...@@ -19,6 +19,8 @@ remote_max_retries: 50 #!final ...@@ -19,6 +19,8 @@ remote_max_retries: 50 #!final
remote_read_timeout_secs: 180.0 #!final remote_read_timeout_secs: 180.0 #!final
channels: channels:
- defaults - defaults
conda_build: #!final
pkg_format: '2'
""" """
_SERVER = "http://www.idiap.ch" _SERVER = "http://www.idiap.ch"
...@@ -216,6 +218,8 @@ def merge_conda_cache(cache, prefix, name): ...@@ -216,6 +218,8 @@ def merge_conda_cache(cache, prefix, name):
# move packages on cache/pkgs to pkgs_dir # move packages on cache/pkgs to pkgs_dir
cached_pkgs_dir = os.path.join(cache, "pkgs") cached_pkgs_dir = os.path.join(cache, "pkgs")
cached_packages = glob.glob(os.path.join(cached_pkgs_dir, "*.tar.bz2")) cached_packages = glob.glob(os.path.join(cached_pkgs_dir, "*.tar.bz2"))
cached_packages.extend(glob.glob(os.path.join(cached_pkgs_dir, "*.conda")))
cached_packages = [ cached_packages = [
k for k in cached_packages if not k.startswith(name + "-") k for k in cached_packages if not k.startswith(name + "-")
] ]
......
...@@ -65,8 +65,8 @@ def should_skip_build(metadata_tuples): ...@@ -65,8 +65,8 @@ def should_skip_build(metadata_tuples):
def next_build_number(channel_url, basename): def next_build_number(channel_url, basename):
"""Calculates the next build number of a package given the channel. """Calculates the next build number of a package given the channel.
This function returns the next build number (integer) for a package given its This function returns the next build number (integer) for a package given
resulting tarball base filename (can be obtained with its resulting tarball base filename (can be obtained with
:py:func:`get_output_path`). :py:func:`get_output_path`).
...@@ -88,8 +88,14 @@ def next_build_number(channel_url, basename): ...@@ -88,8 +88,14 @@ def next_build_number(channel_url, basename):
logger.debug("Downloading channel index from %s", channel_url) logger.debug("Downloading channel index from %s", channel_url)
index = get_index(channel_urls=[channel_url], prepend=False) index = get_index(channel_urls=[channel_url], prepend=False)
# remove .tar.bz2 from name, then split from the end twice, on '-' # remove .tar.bz2/.conda from name, then split from the end twice, on '-'
name, version, build = basename[:-8].rsplit("-", 2) if basename.endswith('.tar.bz2'):
name, version, build = basename[:-8].rsplit("-", 2)
elif basename.endswith('.conda'):
name, version, build = basename[:-6].rsplit("-", 2)
else:
raise RuntimeError("Package name %s does not end in either " \
".tar.bz2 or .conda" % (basename,))
# remove the build number as we're looking for the next value # remove the build number as we're looking for the next value
# examples to be coped with: # examples to be coped with:
...@@ -206,7 +212,8 @@ def get_parsed_recipe(metadata): ...@@ -206,7 +212,8 @@ def get_parsed_recipe(metadata):
def exists_on_channel(channel_url, basename): def exists_on_channel(channel_url, basename):
"""Checks on the given channel if a package with the specs exist. """Checks on the given channel if a package with the specs exist.
This procedure always ignores the package hash code, if one is set This procedure always ignores the package hash code, if one is set. It
differentiates between `.conda` and `.tar.bz2` packages.
Args: Args:
...@@ -214,16 +221,23 @@ def exists_on_channel(channel_url, basename): ...@@ -214,16 +221,23 @@ def exists_on_channel(channel_url, basename):
channel) channel)
basename: The basename of the tarball to search for basename: The basename of the tarball to search for
Returns: A complete package url, if the package already exists in the channel Returns: A complete package url, if the package already exists in the
or ``None`` otherwise. channel or ``None`` otherwise.
""" """
build_number, urls = next_build_number(channel_url, basename) build_number, urls = next_build_number(channel_url, basename)
def _get_build_number(name): def _get_build_number(name):
# remove .tar.bz2 from name, then split from the end twice, on '-' # remove .tar.bz2/.conda from name, then split from the end twice, on
name, version, build = name[:-8].rsplit("-", 2) # '-'
if name.endswith('.conda'):
name, version, build = name[:-6].rsplit("-", 2)
elif name.endswith('.tar.bz2'):
name, version, build = name[:-8].rsplit("-", 2)
else:
raise RuntimeError("Package name %s does not end in either " \
".tar.bz2 or .conda" % (name,))
# remove the build number as we're looking for the next value # remove the build number as we're looking for the next value
# examples to be coped with: # examples to be coped with:
...@@ -239,9 +253,10 @@ def exists_on_channel(channel_url, basename): ...@@ -239,9 +253,10 @@ def exists_on_channel(channel_url, basename):
other_build_numbers = [_get_build_number(os.path.basename(k)) for k in urls] other_build_numbers = [_get_build_number(os.path.basename(k)) for k in urls]
if self_build_number in other_build_numbers: if self_build_number in other_build_numbers:
return "".join( candidate = urls[other_build_numbers.index(self_build_number)]
(channel_url, urls[other_build_numbers.index(self_build_number)]) pkg_type = '.conda' if basename.endswith('.conda') else '.tar.bz2'
) if candidate.endswith(pkg_type): #match
return "".join(channel_url, candidate)
def remove_pins(deps): def remove_pins(deps):
......
...@@ -43,6 +43,7 @@ build: ...@@ -43,6 +43,7 @@ build:
expire_in: 1 week expire_in: 1 week
paths: paths:
- sphinx - sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
tags: tags:
- docker - docker
......
...@@ -46,6 +46,7 @@ stages: ...@@ -46,6 +46,7 @@ stages:
image: continuumio/conda-concourse-ci image: continuumio/conda-concourse-ci
artifacts: artifacts:
paths: paths:
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
...@@ -55,6 +56,7 @@ stages: ...@@ -55,6 +56,7 @@ stages:
- macosx - macosx
artifacts: artifacts:
paths: paths:
- ${CONDA_ROOT}/conda-bld/osx-64/*.conda
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
...@@ -92,6 +94,7 @@ build_linux_37: ...@@ -92,6 +94,7 @@ build_linux_37:
paths: paths:
- dist/*.zip - dist/*.zip
- sphinx - sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.conda
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2 - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache: cache:
key: "build-py37" key: "build-py37"
......
...@@ -121,10 +121,13 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True, includes=None): ...@@ -121,10 +121,13 @@ def remove_old_beta_packages(client, path, dry_run, pyver=True, includes=None):
if f.startswith("."): if f.startswith("."):
continue continue
if not f.endswith(".tar.bz2"):
continue
name, version, build_string = f[:-8].rsplit("-", 2) if f.endswith(".tar.bz2"):
name, version, build_string = f[:-8].rsplit("-", 2)
elif f.endswith(".conda"):
name, version, build_string = f[:-6].rsplit("-", 2)
else:
continue
# see if this package should be included or not in our clean-up # see if this package should be included or not in our clean-up
if (includes is not None) and (not includes.match(name)): if (includes is not None) and (not includes.match(name)):
......
...@@ -80,10 +80,10 @@ def base_deploy(dry_run): ...@@ -80,10 +80,10 @@ def base_deploy(dry_run):
# afterwards) # afterwards)
for arch in ("linux-64", "osx-64", "noarch"): for arch in ("linux-64", "osx-64", "noarch"):
# finds conda dependencies and uploads what we can find # finds conda dependencies and uploads what we can find
package_path = os.path.join( base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
os.environ["CONDA_ROOT"], "conda-bld", arch, "*.tar.bz2" conda_paths = os.path.join(base_path, "*.conda")
) tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
deploy_packages = glob.glob(package_path) deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
for k in deploy_packages: for k in deploy_packages:
...@@ -167,10 +167,11 @@ def deploy(latest, dry_run): ...@@ -167,10 +167,11 @@ def deploy(latest, dry_run):
# afterwards) # afterwards)
for arch in ("linux-64", "osx-64", "noarch"): for arch in ("linux-64", "osx-64", "noarch"):
# finds conda packages and uploads what we can find # finds conda packages and uploads what we can find
package_path = os.path.join( base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
os.environ["CONDA_ROOT"], "conda-bld", arch, name + "*.tar.bz2" conda_paths = os.path.join(base_path, "*.conda")
) tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
deploy_packages = glob.glob(package_path) deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
for k in deploy_packages: for k in deploy_packages:
deploy_conda_package( deploy_conda_package(
k, k,
...@@ -504,16 +505,13 @@ def test(ctx, dry_run): ...@@ -504,16 +505,13 @@ def test(ctx, dry_run):
from .test import test from .test import test
base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", "*",
os.environ["CI_PROJECT_NAME"])
ctx.invoke( ctx.invoke(
test, test,
package=glob.glob( package=glob.glob(base_path + "*.conda") + \
os.path.join( glob.glob(base_path + "*.tar.bz2"),
os.environ["CONDA_ROOT"],
"conda-bld",
"*",
os.environ["CI_PROJECT_NAME"] + "*.tar.bz2",
)
),
condarc=condarc, condarc=condarc,
config=variants_file, config=variants_file,
append_file=append_file, append_file=append_file,
......
...@@ -37,13 +37,13 @@ Examples: ...@@ -37,13 +37,13 @@ Examples:
1. Tests conda package: 1. Tests conda package:
\b \b
$ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2 $ bdt test -vv /path/to/conda-package-v1.0.0.conda
2. Tests multiple conda packages, one after the other: 2. Tests multiple conda packages, one after the other:
\b \b
$ bdt test -vv /path/to/conda-package-v1.0.0.tar.bz2 /path/to/other-conda-package-v2.0.0.tar.bz2 $ bdt test -vv /path/to/conda-package-v1.0.0.conda /path/to/other-conda-package-v2.0.0.conda
""" """
) )
......
{% set name = "python-gitlab" %} {% set name = "python-gitlab" %}
{% set version = "1.10.0" %} {% set version = "1.12.1" %}
package: package:
name: {{ name|lower }} name: {{ name|lower }}
...@@ -7,7 +7,7 @@ package: ...@@ -7,7 +7,7 @@ package:
source: source:
url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
sha256: c4e91b9367c54c3049d702c35dd966f9e5a8989dae1be56ef7a1c35c2b235a58 sha256: 984e110c1f76fd939652c30ce3101267a7064e34417cbfc4687e6106d4db54ec
build: build:
number: 0 number: 0
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment