diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ec7e0f754be049fcaffbd52e713f51a4ad4b1d0..89ed30711b73ff8ce160031f6adb7880110b66a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,6 +56,12 @@ build_linux_36: <<: *linux_build_job variables: PYTHON_VERSION: "3.6" + + +build_linux_37: + <<: *linux_build_job + variables: + PYTHON_VERSION: "3.7" BUILD_EGG: "true" script: - python3 ./bob/devtools/bootstrap.py -vv build @@ -76,18 +82,27 @@ build_macosx_36: PYTHON_VERSION: "3.6" +build_macosx_37: + <<: *macosx_build_job + variables: + PYTHON_VERSION: "3.7" + + # Deploy targets .deploy_template: &deploy_job stage: deploy script: - - python3 ./bob/devtools/bootstrap.py -vv local bdt + - python3 ./bob/devtools/bootstrap.py -vv local base - source ${CONDA_ROOT}/etc/profile.d/conda.sh - - conda activate bdt + - conda activate base + - bdt ci base-deploy -vv - bdt ci deploy -vv - bdt ci clean -vv dependencies: - build_linux_36 + - build_linux_37 - build_macosx_36 + - build_macosx_37 tags: - docker cache: &build_caches @@ -121,14 +136,16 @@ pypi: except: - branches script: - - python3 ./bob/devtools/bootstrap.py -vv local bdt + - python3 ./bob/devtools/bootstrap.py -vv local base - source ${CONDA_ROOT}/etc/profile.d/conda.sh - - conda activate bdt + - conda activate base - bdt ci pypi -vv dist/*.zip - bdt ci clean -vv dependencies: - build_linux_36 + - build_linux_37 - build_macosx_36 + - build_macosx_37 tags: - docker cache: &build_caches diff --git a/bob/devtools/build.py b/bob/devtools/build.py index ab73d6f87d5ca605cdb00b311724644089b05dae..a5b32a9a91572ee9032addf37a02fdb216a1447b 100644 --- a/bob/devtools/build.py +++ b/bob/devtools/build.py @@ -154,6 +154,43 @@ def get_parsed_recipe(metadata): return yaml.load(output) +def exists_on_channel(channel_url, name, version, build_number): + """Checks on the given channel if a package with the specs exist + + Args: + + channel_url: The URL where to look for packages clashes (normally a beta + channel) + name: The name of the package + version: The version of the package + build_number: The build number of the package + + Returns: ``True``, if the package already exists in the channel or ``False`` + otherwise + + """ + + from conda.exports import get_index + + # get the channel index + logger.debug('Downloading channel index from %s', channel_url) + index = get_index(channel_urls=[channel_url], prepend=False) + + logger.info('Checking for %s-%s_*_%s...', name, version, build_number) + for dist in index: + + if dist.name == name and dist.version == version and \ + dist.build_string.endswith('_%s' % build_number): + match = re.match('py[2-9][0-9]+', dist.build_string) + logger.info('Found matching package (%s-%s_%s)', dist.name, dist.version, + dist.build_string) + return True + + logger.info('No matches for %s-%s_%s found among %d packages', + name, version, build_number, len(index)) + return False + + def remove_pins(deps): return [l.split()[0] for l in deps] @@ -407,6 +444,52 @@ def git_clean_build(runner, verbose): ['--exclude=%s' % k for k in exclude_from_cleanup]) +def base_build(server, intranet, recipe_dir, config): + '''Builds a non-beat/bob software dependence that does not exist on defaults + + This function will build a software dependence that is required for our + software stack, but does not (yet) exist on the defaults channels. It first + check if the build should run for the current architecture, checks if the + package is not already built on our public channel and, if that is true, then + proceeds with the build of the dependence. + + + Args: + + server: The base address of the server containing our conda channels + intranet: Boolean indicating if we should add "private"/"public" prefixes + on the returned paths + recipe_dir: The directory containing the recipe's ``meta.yaml`` file + config: A dictionary containing the merged configuration, as produced by + conda-build API's ``get_or_merge_config()`` function + + ''' + + # if you get to this point, tries to build the package + public_channel = bootstrap.get_channels(public=True, stable=True, + server=server, intranet=intranet)[0] + metadata = get_rendered_metadata(recipe_dir, config) + recipe = get_parsed_recipe(metadata) + + if recipe is None: + logger.warn('Skipping build for %s - rendering returned None', recipe_dir) + continue + + if exists_on_channel(public_channel, recipe['package']['name'], + recipe['package']['version'], recipe['build']['number']): + logger.warn('Skipping build for %s-%s_%s - exists on channel already', + recipe['package']['name'], recipe['package']['version'], + recipe['build']['number']) + continue + + # if you get to this point, just builds the package + arch = conda_arch() + logger.info('Building %s-%s (build: %d) for %s', + recipe['package']['name'], recipe['package']['version'], + recipe['build']['number'], arch) + conda_build.api.build(recipe_dir, config=conda_config) + + if __name__ == '__main__': import argparse @@ -477,7 +560,8 @@ if __name__ == '__main__': condarc_options = yaml.load(f) # notice this condarc typically will only contain the defaults channel - we - # need to boost this up with more channels to get it right. + # need to boost this up with more channels to get it right for this package's + # build public = ( args.visibility == 'public' ) channels = bootstrap.get_channels(public=public, stable=(not is_prerelease), server=server, intranet=(not args.internet)) @@ -492,6 +576,15 @@ if __name__ == '__main__': conda_config = make_conda_config(conda_build_config, args.python_version, recipe_append, condarc_options) + # builds all dependencies in the 'deps' subdirectory - or at least checks + # these dependencies are already available; these dependencies go directly to + # the public channel once built + for recipe in glob.glob(os.path.join('deps', '*')): + if not os.path.exists(os.path.join(recipe, 'meta.yaml')): + # ignore - not a conda package + continue + base_build(server, not args.internet, recipe, conda_config) + # retrieve the current build number for this build build_number, _ = next_build_number(channels[0], args.name, version, args.python_version) @@ -517,4 +610,4 @@ if __name__ == '__main__': else: logger.info('twine check (a.k.a. readme check) %s: OK', package[0]) - git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 2)) + git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 3)) diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py index f05c4a425591f90621dbee407d7e77f2e6e0efef..ced320084cae86ffdb8fb24996747f842e509707 100644 --- a/bob/devtools/scripts/ci.py +++ b/bob/devtools/scripts/ci.py @@ -30,6 +30,76 @@ def ci(): pass +@ci.command(epilog=''' +Examples: + + 1. Deploys base build artifacts (dependencies) to the appropriate channels: + + $ bdt ci base-deploy -vv + +''') +@click.option('-d', '--dry-run/--no-dry-run', default=False, + help='Only goes through the actions, but does not execute them ' \ + '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \ + 'printing to help you understand what will be done') +@verbosity_option() +@bdt.raise_on_error +def base_deploy(dry_run): + """Deploys dependencies not available at the defaults channel + + Deployment happens to our public channel directly, as these are + dependencies are required for proper bob/beat package runtime environments. + """ + + if dry_run: + logger.warn('!!!! DRY RUN MODE !!!!') + logger.warn('Nothing is being deployed to server') + + package = os.environ['CI_PROJECT_PATH'] + + from ..constants import WEBDAV_PATHS + server_info = WEBDAV_PATHS[True][True] #stable=True, visible=True + + logger.info('Deploying dependence packages to %s%s%s...', SERVER, + server_info['root'], server_info['conda']) + + # setup webdav connection + webdav_options = { + 'webdav_hostname': SERVER, + 'webdav_root': server_info['root'], + 'webdav_login': os.environ['DOCUSER'], + 'webdav_password': os.environ['DOCPASS'], + } + from ..webdav3 import client as webdav + davclient = webdav.Client(webdav_options) + assert davclient.valid() + + group, name = package.split('/') + + # uploads conda package artificats + for arch in ('linux-64', 'osx-64', 'noarch'): + # finds conda dependencies and uploads what we can find + package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch, + '*.tar.bz2') + deploy_packages = glob.glob(package_path) + for k in deploy_packages: + basename = os.path.basename(k) + if basename.startswith(name): + logger.debug('Skipping deploying of %s - not a base package', k) + continue + + remote_path = '%s/%s/%s' % (server_info['conda'], arch, basename) + if davclient.check(remote_path): + raise RuntimeError('The file %s/%s already exists on the server ' \ + '- this can be due to more than one build with deployment ' \ + 'running at the same time. Re-running the broken builds ' \ + 'normally fixes it' % (SERVER, remote_path)) + logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'], + remote_path) + if not dry_run: + davclient.upload(local_path=k, remote_path=remote_path) + + @ci.command(epilog=''' Examples: @@ -291,4 +361,4 @@ def clean(ctx): from ..build import git_clean_build from ..bootstrap import run_cmdline - git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=2)) + git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=3)) diff --git a/conda/meta.yaml b/conda/meta.yaml index deb63262b5fc1e768835a2139cd3a9bf9dec0f88..0cc376da3ab5a2eedcc71f43480f49f8bad98279 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -68,6 +68,7 @@ test: - bdt new --help - bdt ci --help - bdt ci build --help + - bdt ci base-deploy --help - bdt ci deploy --help - bdt ci pypi --help - bdt ci readme --help diff --git a/deps/python-gitlab/meta.yaml b/deps/python-gitlab/meta.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cce6e8296470bc5ea3d891f0ef6436c48ded162 --- /dev/null +++ b/deps/python-gitlab/meta.yaml @@ -0,0 +1,46 @@ +{% set name = "python-gitlab" %} +{% set version = "1.7.0" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz + sha256: 401ef8929db4dcb5b08e0a2263a0a70599fc7e5b27615f956ac26d245802d09e + +build: + number: 0 + script: "{{ PYTHON }} -m pip install . -vv" + entry_points: + - gitlab = gitlab.cli:main + +requirements: + host: + - python + - pip + run: + - python + - requests + - six + +test: + imports: + - gitlab + +about: + home: https://github.com/python-gitlab/python-gitlab + license: LGPL-3.0 + license_family: LGPL + license_file: COPYING + summary: 'Python wrapper for the GitLab API' + description: | + python-gitlab is a Python package providing access to the GitLab + server API. It supports the v4 API of GitLab, and provides a CLI + tool (gitlab). + doc_url: https://python-gitlab.readthedocs.io/ + dev_url: https://github.com/python-gitlab/python-gitlab + +extra: + recipe-maintainers: + - anjos diff --git a/setup.py b/setup.py index 178fd46511dc7f630499511bcee83ddd0b802bd6..01918fb2ffcbeb365f69dbfa00430be17ff57aa5 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ setup( 'bdt.ci.cli': [ 'build = bob.devtools.scripts.ci:build', 'clean = bob.devtools.scripts.ci:clean', + 'base-deploy = bob.devtools.scripts.ci:base_deploy', 'deploy = bob.devtools.scripts.ci:deploy', 'readme = bob.devtools.scripts.ci:readme', 'pypi = bob.devtools.scripts.ci:pypi',